From 8c56e4516e55c7ed9c14779f0e77e00f055d9a81 Mon Sep 17 00:00:00 2001
From: vozerov-gridgain <vozerov@gridgain.com>
Date: Fri, 2 Sep 2016 18:05:16 +0300
Subject: [PATCH 001/516] IGNITE-3827: Removed double marshalling of keys in
 DataStreamerImpl.addData(Map) method.

---
 .../datastreamer/DataStreamerImpl.java        | 23 ++++++++-----------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index e565cbaa43676..a3bae249d13ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -87,7 +87,6 @@
 import org.apache.ignite.internal.util.lang.GridPeerDeployAware;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.A;
@@ -513,23 +512,19 @@ else if (autoFlushFreq == 0)
 
             activeFuts.add(resFut);
 
-            Collection<KeyCacheObject> keys = null;
+            Collection<KeyCacheObject> keys =
+                new GridConcurrentHashSet<>(entries.size(), U.capacity(entries.size()), 1);
 
-            if (entries.size() > 1) {
-                keys = new GridConcurrentHashSet<>(entries.size(), U.capacity(entries.size()), 1);
+            Collection<DataStreamerEntry> entries0 = new ArrayList<>(entries.size());
 
-                for (Map.Entry<K, V> entry : entries)
-                    keys.add(cacheObjProc.toCacheKeyObject(cacheObjCtx, null, entry.getKey(), true));
-            }
+            for (Map.Entry<K, V> entry : entries) {
+                KeyCacheObject key = cacheObjProc.toCacheKeyObject(cacheObjCtx, null, entry.getKey(), true);
+                CacheObject val = cacheObjProc.toCacheObject(cacheObjCtx, entry.getValue(), true);
 
-            Collection<? extends DataStreamerEntry> entries0 = F.viewReadOnly(entries, new C1<Entry<K, V>, DataStreamerEntry>() {
-                @Override public DataStreamerEntry apply(Entry<K, V> e) {
-                    KeyCacheObject key = cacheObjProc.toCacheKeyObject(cacheObjCtx, null, e.getKey(), true);
-                    CacheObject val = cacheObjProc.toCacheObject(cacheObjCtx, e.getValue(), true);
+                keys.add(key);
 
-                    return new DataStreamerEntry(key, val);
-                }
-            });
+                entries0.add(new DataStreamerEntry(key, val));
+            }
 
             load0(entries0, resFut, keys, 0);
 

From 4dc624fc9a8852f77f1fe7db4dc06a474b34c2eb Mon Sep 17 00:00:00 2001
From: vozerov-gridgain <vozerov@gridgain.com>
Date: Fri, 2 Sep 2016 18:23:09 +0300
Subject: [PATCH 002/516] IGNITE-3829: Optimized affinity key field name
 handling.

---
 .../CacheObjectBinaryProcessorImpl.java       | 35 +++++++++++++++----
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 03378743c2ee3..ecd27f7a19af7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.binary.BinaryBasicNameMapper;
+import org.apache.ignite.binary.BinaryField;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryObjectBuilder;
 import org.apache.ignite.binary.BinaryObjectException;
@@ -89,6 +90,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T1;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -159,6 +161,9 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
     /** Metadata updates collected before metadata cache is initialized. */
     private final Map<Integer, BinaryMetadata> metaBuf = new ConcurrentHashMap<>();
 
+    /** Cached affinity key field names. */
+    private final ConcurrentHashMap<Integer, T1<BinaryField>> affKeyFields = new ConcurrentHashMap<>();
+
     /**
      * @param ctx Kernal context.
      */
@@ -684,22 +689,38 @@ public GridBinaryMarshaller marshaller() {
      * @return Affinity key.
      */
     public Object affinityKey(BinaryObject po) {
+        // Fast path for already cached field.
+        if (po instanceof BinaryObjectEx) {
+            int typeId = ((BinaryObjectEx)po).typeId();
+
+            T1<BinaryField> fieldHolder = affKeyFields.get(typeId);
+
+            if (fieldHolder != null) {
+                BinaryField field = fieldHolder.get();
+
+                return field != null ? field.value(po) : po;
+            }
+        }
+
+        // Slow path if affinity field is not cached yet.
         try {
             BinaryType meta = po instanceof BinaryObjectEx ? ((BinaryObjectEx)po).rawType() : po.type();
 
             if (meta != null) {
-                String affKeyFieldName = meta.affinityKeyFieldName();
+                String name = meta.affinityKeyFieldName();
+
+                affKeyFields.putIfAbsent(meta.typeId(), new T1<>(meta.field(name)));
 
-                if (affKeyFieldName != null)
-                    return po.field(affKeyFieldName);
+                if (name != null)
+                    return po.field(name);
             }
             else if (po instanceof BinaryObjectEx) {
-                int id = ((BinaryObjectEx)po).typeId();
+                int typeId = ((BinaryObjectEx)po).typeId();
 
-                String affKeyFieldName = binaryCtx.affinityKeyFieldName(id);
+                String name = binaryCtx.affinityKeyFieldName(typeId);
 
-                if (affKeyFieldName != null)
-                    return po.field(affKeyFieldName);
+                if (name != null)
+                    return po.field(name);
             }
         }
         catch (BinaryObjectException e) {

From 7b0d2326ccc62afd5d162b056398f96d8d7c9100 Mon Sep 17 00:00:00 2001
From: vozerov-gridgain <vozerov@gridgain.com>
Date: Mon, 5 Sep 2016 10:19:48 +0300
Subject: [PATCH 003/516] IGNITE-3829: Additional fix.

---
 .../cache/binary/CacheObjectBinaryProcessorImpl.java  | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index ecd27f7a19af7..82e67aca1518a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -709,10 +709,15 @@ public Object affinityKey(BinaryObject po) {
             if (meta != null) {
                 String name = meta.affinityKeyFieldName();
 
-                affKeyFields.putIfAbsent(meta.typeId(), new T1<>(meta.field(name)));
+                if (name != null) {
+                    BinaryField field = meta.field(name);
 
-                if (name != null)
-                    return po.field(name);
+                    affKeyFields.putIfAbsent(meta.typeId(), new T1<>(field));
+
+                    return field.value(po);
+                }
+                else
+                    affKeyFields.putIfAbsent(meta.typeId(), new T1<BinaryField>(null));
             }
             else if (po instanceof BinaryObjectEx) {
                 int typeId = ((BinaryObjectEx)po).typeId();

From b8cb82de65a529040ea18b0dc03fa7109c69bb4a Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Thu, 29 Dec 2016 14:48:45 +0700
Subject: [PATCH 004/516] IGNITE-4442 Implemented cache affinity configuration.

(cherry picked from commit f4a1e6c)
---
 modules/web-console/backend/app/mongo.js      | 19 +++++
 .../generator/AbstractTransformer.js          |  5 ++
 .../modules/configuration/generator/Beans.js  |  4 +
 .../generator/ConfigurationGenerator.js       | 36 ++++++++
 .../states/configuration/caches/affinity.jade | 82 +++++++++++++++++++
 .../states/configuration/caches/memory.jade   |  4 +-
 .../frontend/views/configuration/caches.jade  |  1 +
 7 files changed, 149 insertions(+), 2 deletions(-)
 create mode 100644 modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade

diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index 58ab11983aca1..dd71f3a8bf57e 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -140,6 +140,25 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
         cacheMode: {type: String, enum: ['PARTITIONED', 'REPLICATED', 'LOCAL']},
         atomicityMode: {type: String, enum: ['ATOMIC', 'TRANSACTIONAL']},
 
+        affinity: {
+            kind: {type: String, enum: ['Default', 'Rendezvous', 'Fair', 'Custom']},
+            Rendezvous: {
+                affinityBackupFilter: String,
+                partitions: Number,
+                excludeNeighbors: Boolean
+            },
+            Fair: {
+                affinityBackupFilter: String,
+                partitions: Number,
+                excludeNeighbors: Boolean
+            },
+            Custom: {
+                className: String
+            }
+        },
+
+        affinityMapper: String,
+
         nodeFilter: {
             kind: {type: String, enum: ['Default', 'Exclude', 'IGFS', 'OnNodes', 'Custom']},
             Exclude: {
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
index f5afe59805114..40d937e1d39dd 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
@@ -210,6 +210,11 @@ export default class AbstractTransformer {
         return this.toSection(this.generator.cacheGeneral(cache));
     }
 
+    // Generate cache memory group.
+    static cacheAffinity(cache) {
+        return this.toSection(this.generator.cacheAffinity(cache));
+    }
+
     // Generate cache memory group.
     static cacheMemory(cache) {
         return this.toSection(this.generator.cacheMemory(cache));
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Beans.js b/modules/web-console/frontend/app/modules/configuration/generator/Beans.js
index ca1934282510e..0972eace6801c 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Beans.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Beans.js
@@ -116,6 +116,10 @@ export class Bean extends EmptyBean {
         return this._property(this.arguments, 'int', model, null, _.nonNil);
     }
 
+    boolConstructorArgument(model) {
+        return this._property(this.arguments, 'boolean', model, null, _.nonNil);
+    }
+
     classConstructorArgument(model) {
         return this._property(this.arguments, 'java.lang.Class', model, null, _.nonEmpty);
     }
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 8770bf6063c73..de2b750675fd8 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -1453,6 +1453,41 @@ export default class IgniteConfigurationGenerator {
         return ccfg;
     }
 
+    // Generation of constructor for affinity function.
+    static cacheAffinityFunction(cls, func) {
+        const affBean = new Bean(cls, 'affinityFunction', func);
+
+        affBean.boolConstructorArgument('excludeNeighbors')
+            .intProperty('partitions')
+            .emptyBeanProperty('affinityBackupFilter');
+
+        return affBean;
+    }
+
+    // Generate cache memory group.
+    static cacheAffinity(cache, ccfg = this.cacheConfigurationBean(cache)) {
+        switch (_.get(cache, 'affinity.kind')) {
+            case 'Rendezvous':
+                ccfg.beanProperty('affinity', this.cacheAffinityFunction('org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction', cache.affinity.Rendezvous));
+
+                break;
+            case 'Fair':
+                ccfg.beanProperty('affinity', this.cacheAffinityFunction('org.apache.ignite.cache.affinity.fair.FairAffinityFunction', cache.affinity.Fair));
+
+                break;
+            case 'Custom':
+                ccfg.emptyBeanProperty('affinity.Custom.className', 'affinity');
+
+                break;
+            default:
+                // No-op.
+        }
+
+        ccfg.emptyBeanProperty('affinityMapper');
+
+        return ccfg;
+    }
+
     // Generate cache memory group.
     static cacheMemory(cache, ccfg = this.cacheConfigurationBean(cache)) {
         ccfg.enumProperty('memoryMode');
@@ -1728,6 +1763,7 @@ export default class IgniteConfigurationGenerator {
 
     static cacheConfiguration(cache, ccfg = this.cacheConfigurationBean(cache)) {
         this.cacheGeneral(cache, ccfg);
+        this.cacheAffinity(cache, ccfg);
         this.cacheMemory(cache, ccfg);
         this.cacheQuery(cache, cache.domains, ccfg);
         this.cacheStore(cache, cache.domains, ccfg);
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade
new file mode 100644
index 0000000000000..3c4746bbca794
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade
@@ -0,0 +1,82 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins.jade
+
+-var form = 'affinity'
+-var model = 'backupItem'
+-var affModel = model + '.affinity'
+-var affMapModel = model + '.affinityMapper'
+-var rendezvousAff = affModel + '.kind === "Rendezvous"'
+-var fairAff = affModel + '.kind === "Fair"'
+-var customAff = affModel + '.kind === "Custom"'
+-var customAffMapper = affMapModel + '.kind === "Custom"'
+-var rendPartitionsRequired = rendezvousAff + ' && ' + affModel + '.Rendezvous.affinityBackupFilter'
+-var fairPartitionsRequired = fairAff + ' && ' + affModel + '.Fair.affinityBackupFilter'
+
+.panel.panel-default(ng-form=form novalidate)
+    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+        ignite-form-panel-chevron
+        label Affinity Collocation
+        ignite-form-field-tooltip.tipLabel
+            | Collocate data with data to improve performance and scalability of your application#[br]
+            | #[a(href="http://apacheignite.gridgain.org/docs/affinity-collocation" target="_blank") More info]
+        ignite-form-revert
+    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
+        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+            .col-sm-6
+                .settings-row
+                    +dropdown('Function:', affModel + '.kind', '"AffinityKind"', 'true', 'Default',
+                        '[\
+                            {value: "Rendezvous", label: "Rendezvous"},\
+                            {value: "Fair", label: "Fair"},\
+                            {value: "Custom", label: "Custom"},\
+                            {value: undefined, label: "Default"}\
+                        ]',
+                        'Key topology resolver to provide mapping from keys to nodes\
+                        <ul>\
+                            <li>Rendezvous - Based on Highest Random Weight algorithm<br/></li>\
+                            <li>Fair - Tries to ensure that all nodes get equal number of partitions with minimum amount of reassignments between existing nodes<br/></li>\
+                            <li>Custom - Custom implementation of key affinity fynction<br/></li>\
+                            <li>Default - By default rendezvous affinity function  with 1024 partitions is used<br/></li>\
+                        </ul>')
+                .panel-details(ng-if=rendezvousAff)
+                    .details-row
+                        +number-required('Partitions', affModel + '.Rendezvous.partitions', '"RendPartitions"', 'true', rendPartitionsRequired, '1024', '1', 'Number of partitions')
+                    .details-row
+                        +java-class('Backup filter', affModel + '.Rendezvous.affinityBackupFilter', '"RendAffinityBackupFilter"', 'true', 'false',
+                            'Backups will be selected from all nodes that pass this filter')
+                    .details-row
+                        +checkbox('Exclude neighbors', affModel + '.Rendezvous.excludeNeighbors', '"RendExcludeNeighbors"',
+                            'Exclude same - host - neighbors from being backups of each other and specified number of backups')
+                .panel-details(ng-if=fairAff)
+                    .details-row
+                        +number-required('Partitions', affModel + '.Fair.partitions', '"FairPartitions"', 'true', fairPartitionsRequired, '256', '1', 'Number of partitions')
+                    .details-row
+                        +java-class('Backup filter', affModel + '.Fair.affinityBackupFilter', '"FairAffinityBackupFilter"', 'true', 'false',
+                            'Backups will be selected from all nodes that pass this filter')
+                    .details-row
+                        +checkbox('Exclude neighbors', affModel + '.Fair.excludeNeighbors', '"FairExcludeNeighbors"',
+                            'Exclude same - host - neighbors from being backups of each other and specified number of backups')
+                .panel-details(ng-if=customAff)
+                    .details-row
+                        +java-class('Class name:', affModel + '.Custom.className', '"AffCustomClassName"', 'true', customAff,
+                            'Custom key affinity function implementation class name')
+                .settings-row
+                    +java-class('Mapper:', model + '.affinityMapper', '"AffMapCustomClassName"', 'true', 'false',
+                        'Provide custom affinity key for any given key')
+            .col-sm-6
+                +preview-xml-java(model, 'cacheAffinity')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade
index f2d3e2bfa656c..e8dfb3a15a1fc 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade
@@ -61,7 +61,7 @@ include /app/helpers/jade/mixins.jade
                                 Note that in this mode entries can be evicted only to swap\
                             </li>\
                         </ul>')
-                .settings-row(data-ng-show=model + '.memoryMode !== "OFFHEAP_VALUES"')
+                .settings-row(ng-show=model + '.memoryMode !== "OFFHEAP_VALUES"')
                     +dropdown-required('Off-heap memory:', model + '.offHeapMode', '"offHeapMode"', 'true',
                         model + '.memoryMode === "OFFHEAP_TIERED"',
                         'Disabled',
@@ -76,7 +76,7 @@ include /app/helpers/jade/mixins.jade
                             <li>Limited - Off-heap storage has limited size</li>\
                             <li>Unlimited - Off-heap storage grow infinitely (it is up to user to properly add and remove entries from cache to ensure that off-heap storage does not grow infinitely)</li>\
                         </ul>')
-                .settings-row(data-ng-if=model + '.offHeapMode === 1 && ' + model + '.memoryMode !== "OFFHEAP_VALUES"')
+                .settings-row(ng-if=model + '.offHeapMode === 1 && ' + model + '.memoryMode !== "OFFHEAP_VALUES"')
                     +number-required('Off-heap memory max size:', model + '.offHeapMaxMemory', '"offHeapMaxMemory"', 'true',
                         model + '.offHeapMode === 1', 'Enter off-heap memory size', '1',
                         'Maximum amount of memory available to off-heap storage in bytes')
diff --git a/modules/web-console/frontend/views/configuration/caches.jade b/modules/web-console/frontend/views/configuration/caches.jade
index 4a4cf2e1a1d24..73a6309d223c6 100644
--- a/modules/web-console/frontend/views/configuration/caches.jade
+++ b/modules/web-console/frontend/views/configuration/caches.jade
@@ -44,6 +44,7 @@ include /app/helpers/jade/mixins.jade
                         +advanced-options-toggle-default
 
                         div(ng-show='ui.expanded')
+                            include /app/modules/states/configuration/caches/affinity.jade
                             include /app/modules/states/configuration/caches/concurrency.jade
                             include /app/modules/states/configuration/caches/near-cache-client.jade
                             include /app/modules/states/configuration/caches/near-cache-server.jade

From 71412cecd861119965a873520da96078f99c94e2 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Fri, 30 Dec 2016 13:41:34 +0300
Subject: [PATCH 005/516] IGNITE-4424 REPLICATED cache isn't synced across
 nodes

---
 .../GridNearAtomicAbstractUpdateFuture.java   |  34 ++-
 .../GridNearAtomicSingleUpdateFuture.java     |  44 ++--
 .../atomic/GridNearAtomicUpdateFuture.java    |  57 ++---
 .../AtomicPutAllChangingTopologyTest.java     | 212 ++++++++++++++++++
 .../IgniteCacheFailoverTestSuite.java         |   3 +
 5 files changed, 284 insertions(+), 66 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
index 2fbabaa54fc9c..c92e0f5ec5b42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
@@ -212,14 +212,18 @@ public void map() {
             // Cannot remap.
             remapCnt = 1;
 
-            map(topVer);
+            GridCacheVersion futVer = addAtomicFuture(topVer);
+
+            if (futVer != null)
+                map(topVer, futVer);
         }
     }
 
     /**
      * @param topVer Topology version.
+     * @param futVer Future version
      */
-    protected abstract void map(AffinityTopologyVersion topVer);
+    protected abstract void map(AffinityTopologyVersion topVer, GridCacheVersion futVer);
 
     /**
      * Maps future on ready topology.
@@ -302,7 +306,7 @@ protected void mapSingle(UUID nodeId, GridNearAtomicAbstractUpdateRequest req) {
      * @param req Request.
      * @param e Error.
      */
-    protected void onSendError(GridNearAtomicAbstractUpdateRequest req, IgniteCheckedException e) {
+    protected final void onSendError(GridNearAtomicAbstractUpdateRequest req, IgniteCheckedException e) {
         synchronized (mux) {
             GridNearAtomicUpdateResponse res = new GridNearAtomicUpdateResponse(cctx.cacheId(),
                 req.nodeId(),
@@ -314,4 +318,28 @@ protected void onSendError(GridNearAtomicAbstractUpdateRequest req, IgniteChecke
             onResult(req.nodeId(), res, true);
         }
     }
+
+    /**
+     * Adds future prevents topology change before operation complete.
+     * Should be invoked before topology lock released.
+     *
+     * @param topVer Topology version.
+     * @return Future version in case future added.
+     */
+    protected final GridCacheVersion addAtomicFuture(AffinityTopologyVersion topVer) {
+        GridCacheVersion futVer = cctx.versions().next(topVer);
+
+        synchronized (mux) {
+            assert this.futVer == null : this;
+            assert this.topVer == AffinityTopologyVersion.ZERO : this;
+
+            this.topVer = topVer;
+            this.futVer = futVer;
+        }
+
+        if (storeFuture() && !cctx.mvcc().addAtomicFuture(futVer, this))
+            return null;
+
+        return futVer;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
index bd231cf0003d8..7376affca4455 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
@@ -348,14 +348,7 @@ else if (res.error() != null) {
                 @Override public void apply(final IgniteInternalFuture<AffinityTopologyVersion> fut) {
                     cctx.kernalContext().closure().runLocalSafe(new Runnable() {
                         @Override public void run() {
-                            try {
-                                AffinityTopologyVersion topVer = fut.get();
-
-                                map(topVer);
-                            }
-                            catch (IgniteCheckedException e) {
-                                onDone(e);
-                            }
+                            mapOnTopology();
                         }
                     });
                 }
@@ -388,7 +381,9 @@ private void updateNear(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicU
     @Override protected void mapOnTopology() {
         cache.topology().readLock();
 
-        AffinityTopologyVersion topVer = null;
+        AffinityTopologyVersion topVer;
+
+        GridCacheVersion futVer;
 
         try {
             if (cache.topology().stopping()) {
@@ -410,6 +405,8 @@ private void updateNear(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicU
                 }
 
                 topVer = fut.topologyVersion();
+
+                futVer = addAtomicFuture(topVer);
             }
             else {
                 if (waitTopFut) {
@@ -435,11 +432,12 @@ private void updateNear(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicU
             cache.topology().readUnlock();
         }
 
-        map(topVer);
+        if (futVer != null)
+            map(topVer, futVer);
     }
 
     /** {@inheritDoc} */
-    protected void map(AffinityTopologyVersion topVer) {
+    @Override protected void map(AffinityTopologyVersion topVer, GridCacheVersion futVer) {
         Collection<ClusterNode> topNodes = CU.affinityNodes(cctx, topVer);
 
         if (F.isEmpty(topNodes)) {
@@ -449,11 +447,6 @@ protected void map(AffinityTopologyVersion topVer) {
             return;
         }
 
-        Exception err = null;
-        GridNearAtomicAbstractUpdateRequest singleReq0 = null;
-
-        GridCacheVersion futVer = cctx.versions().next(topVer);
-
         GridCacheVersion updVer;
 
         // Assign version on near node in CLOCK ordering mode even if fastMap is false.
@@ -470,16 +463,17 @@ protected void map(AffinityTopologyVersion topVer) {
         else
             updVer = null;
 
+        Exception err = null;
+        GridNearAtomicAbstractUpdateRequest singleReq0 = null;
+
         try {
             singleReq0 = mapSingleUpdate(topVer, futVer, updVer);
 
             synchronized (mux) {
-                assert this.futVer == null : this;
-                assert this.topVer == AffinityTopologyVersion.ZERO : this;
+                assert this.futVer == futVer || (this.isDone() && this.error() != null);
+                assert this.topVer == topVer;
 
-                this.topVer = topVer;
                 this.updVer = updVer;
-                this.futVer = futVer;
 
                 resCnt = 0;
 
@@ -496,14 +490,6 @@ protected void map(AffinityTopologyVersion topVer) {
             return;
         }
 
-        if (storeFuture()) {
-            if (!cctx.mvcc().addAtomicFuture(futVer, this)) {
-                assert isDone() : this;
-
-                return;
-            }
-        }
-
         // Optimize mapping for single key.
         mapSingle(singleReq0.nodeId(), singleReq0);
     }
@@ -511,7 +497,7 @@ protected void map(AffinityTopologyVersion topVer) {
     /**
      * @return Future version.
      */
-    GridCacheVersion onFutureDone() {
+    private GridCacheVersion onFutureDone() {
         GridCacheVersion ver0;
 
         GridFutureAdapter<Void> fut0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index cd6411725748b..950e5bdd2b9b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -456,14 +456,7 @@ else if (!nodeErr)
                 @Override public void apply(final IgniteInternalFuture<AffinityTopologyVersion> fut) {
                     cctx.kernalContext().closure().runLocalSafe(new Runnable() {
                         @Override public void run() {
-                            try {
-                                AffinityTopologyVersion topVer = fut.get();
-
-                                map(topVer, remapKeys);
-                            }
-                            catch (IgniteCheckedException e) {
-                                onDone(e);
-                            }
+                            mapOnTopology();
                         }
                     });
                 }
@@ -497,7 +490,9 @@ private void updateNear(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdat
     @Override protected void mapOnTopology() {
         cache.topology().readLock();
 
-        AffinityTopologyVersion topVer = null;
+        AffinityTopologyVersion topVer;
+
+        GridCacheVersion futVer;
 
         try {
             if (cache.topology().stopping()) {
@@ -519,6 +514,8 @@ private void updateNear(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdat
                 }
 
                 topVer = fut.topologyVersion();
+
+                futVer = addAtomicFuture(topVer);
             }
             else {
                 if (waitTopFut) {
@@ -544,7 +541,8 @@ private void updateNear(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdat
             cache.topology().readUnlock();
         }
 
-        map(topVer, null);
+        if (futVer != null)
+            map(topVer, futVer, remapKeys);
     }
 
     /**
@@ -602,15 +600,18 @@ private void doUpdate(Map<UUID, GridNearAtomicFullUpdateRequest> mappings) {
     }
 
     /** {@inheritDoc} */
-    protected void map(AffinityTopologyVersion topVer) {
-        map(topVer, null);
+    @Override protected void map(AffinityTopologyVersion topVer, GridCacheVersion futVer) {
+        map(topVer, futVer, null);
     }
 
     /**
      * @param topVer Topology version.
+     * @param futVer Future ID.
      * @param remapKeys Keys to remap.
      */
-    void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> remapKeys) {
+    void map(AffinityTopologyVersion topVer,
+        GridCacheVersion futVer,
+        @Nullable Collection<KeyCacheObject> remapKeys) {
         Collection<ClusterNode> topNodes = CU.affinityNodes(cctx, topVer);
 
         if (F.isEmpty(topNodes)) {
@@ -620,14 +621,6 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
             return;
         }
 
-        Exception err = null;
-        GridNearAtomicFullUpdateRequest singleReq0 = null;
-        Map<UUID, GridNearAtomicFullUpdateRequest> mappings0 = null;
-
-        int size = keys.size();
-
-        GridCacheVersion futVer = cctx.versions().next(topVer);
-
         GridCacheVersion updVer;
 
         // Assign version on near node in CLOCK ordering mode even if fastMap is false.
@@ -644,6 +637,12 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
         else
             updVer = null;
 
+        Exception err = null;
+        GridNearAtomicFullUpdateRequest singleReq0 = null;
+        Map<UUID, GridNearAtomicFullUpdateRequest> mappings0 = null;
+
+        int size = keys.size();
+
         try {
             if (size == 1 && !fastMap) {
                 assert remapKeys == null || remapKeys.size() == 1;
@@ -676,12 +675,10 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
             }
 
             synchronized (mux) {
-                assert this.futVer == null : this;
-                assert this.topVer == AffinityTopologyVersion.ZERO : this;
+                assert this.futVer == futVer || (this.isDone() && this.error() != null);
+                assert this.topVer == topVer;
 
-                this.topVer = topVer;
                 this.updVer = updVer;
-                this.futVer = futVer;
 
                 resCnt = 0;
 
@@ -701,14 +698,6 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
             return;
         }
 
-        if (storeFuture()) {
-            if (!cctx.mvcc().addAtomicFuture(futVer, this)) {
-                assert isDone() : this;
-
-                return;
-            }
-        }
-
         // Optimize mapping for single key.
         if (singleReq0 != null)
             mapSingle(singleReq0.nodeId(), singleReq0);
@@ -725,7 +714,7 @@ void map(AffinityTopologyVersion topVer, @Nullable Collection<KeyCacheObject> re
     /**
      * @return Future version.
      */
-    GridCacheVersion onFutureDone() {
+    private GridCacheVersion onFutureDone() {
         GridCacheVersion ver0;
 
         GridFutureAdapter<Void> fut0;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java
new file mode 100644
index 0000000000000..878cb17b5c3e2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/AtomicPutAllChangingTopologyTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.atomic;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.affinity.fair.FairAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CachePeekMode.BACKUP;
+import static org.apache.ignite.cache.CachePeekMode.PRIMARY;
+import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/** */
+public class AtomicPutAllChangingTopologyTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int NODES_CNT = 3;
+
+    /** */
+    public static final String CACHE_NAME = "test-cache";
+
+    /** */
+    private static final int CACHE_SIZE = 20_000;
+
+    /** */
+    private static volatile CountDownLatch FILLED_LATCH;
+
+    /**
+     * @return Cache configuration.
+     */
+    private CacheConfiguration<Integer, Integer> cacheConfig() {
+        return new CacheConfiguration<Integer, Integer>()
+            .setAtomicityMode(ATOMIC)
+            .setCacheMode(REPLICATED)
+            .setAffinity(new FairAffinityFunction(false, 1))
+            .setWriteSynchronizationMode(FULL_SYNC)
+            .setRebalanceMode(SYNC)
+            .setName(CACHE_NAME);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPutAllOnChangingTopology() throws Exception {
+        List<IgniteInternalFuture> futs = new LinkedList<>();
+
+        for (int i = 1; i < NODES_CNT; i++)
+            futs.add(startNodeAsync(i));
+
+        futs.add(startSeedNodeAsync());
+
+        boolean failed = false;
+
+        for (IgniteInternalFuture fut : futs) {
+            try {
+                fut.get();
+            }
+            catch (Throwable th) {
+                log.error("Check failed.", th);
+
+                failed = true;
+            }
+        }
+
+        if (failed)
+            throw new RuntimeException("Test Failed.");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        FILLED_LATCH = new CountDownLatch(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @return Future.
+     * @throws IgniteCheckedException If failed.
+     */
+    private IgniteInternalFuture startSeedNodeAsync() throws IgniteCheckedException {
+        return GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Boolean call() throws Exception {
+                Ignite node = startGrid(0);
+
+                log.info("Creating cache.");
+
+                IgniteCache<Integer, Integer> cache = node.getOrCreateCache(cacheConfig());
+
+                log.info("Created cache.");
+
+                Map<Integer, Integer> data = new HashMap<>(CACHE_SIZE);
+
+                for (int i = 0; i < CACHE_SIZE; i++)
+                    data.put(i, i);
+
+                log.info("Filling.");
+
+                cache.putAll(data);
+
+                log.info("Filled.");
+
+                FILLED_LATCH.countDown();
+
+                checkCacheState(node, cache);
+
+                return true;
+            }
+        });
+    }
+
+    /**
+     * @param nodeId Node index.
+     * @return Future.
+     * @throws IgniteCheckedException If failed.
+     */
+    private IgniteInternalFuture startNodeAsync(final int nodeId) throws IgniteCheckedException {
+        return GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Boolean call() throws Exception {
+                Ignite node = startGrid(nodeId);
+
+                log.info("Getting cache.");
+
+                IgniteCache<Integer, Integer> cache = node.getOrCreateCache(cacheConfig());
+
+                log.info("Got cache.");
+
+                FILLED_LATCH.await();
+
+                log.info("Got Filled.");
+
+                cache.put(1, nodeId);
+
+                checkCacheState(node, cache);
+
+                return true;
+            }
+        });
+    }
+
+    /**
+     * @param node Node.
+     * @param cache Cache.
+     * @throws Exception If failed.
+     */
+    private void checkCacheState(Ignite node, IgniteCache<Integer, Integer> cache) throws Exception {
+        int locSize = cache.localSize(PRIMARY, BACKUP);
+        int locSize2 = -1;
+
+        if (locSize != CACHE_SIZE) {
+            U.sleep(5000);
+
+            // Rechecking.
+            locSize2 = cache.localSize(PRIMARY, BACKUP);
+        }
+
+        assertEquals("Wrong cache size on node [node=" + node.configuration().getGridName() +
+            ", expected= " + CACHE_SIZE +
+            ", actual=" + locSize +
+            ", actual2=" + locSize2 + "]",
+            locSize, CACHE_SIZE);
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
index c9e507d33e11c..5bc67292b33de 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheTxNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridNearCacheTxNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteAtomicLongChangingTopologySelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.AtomicPutAllChangingTopologyTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicClientInvalidPartitionHandlingSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicClientRemoveFailureTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicInvalidPartitionHandlingSelfTest;
@@ -95,6 +96,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridCacheTxNodeFailureSelfTest.class);
         suite.addTestSuite(GridNearCacheTxNodeFailureSelfTest.class);
 
+        suite.addTestSuite(AtomicPutAllChangingTopologyTest.class);
+
         return suite;
     }
 }

From fe424591161595de1151a29cf1cdeb50456c3e39 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 12 Jan 2017 17:43:21 +0300
Subject: [PATCH 006/516] Minors

---
 .../IgniteCacheExpiryPolicyAbstractTest.java  | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index 794519a0dae48..4368a1533f6d3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -36,24 +36,30 @@
 import javax.cache.expiry.ModifiedExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheMemoryMode;
 import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
+import org.apache.ignite.internal.processors.cache.GridCacheTestStore;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.PAX;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
@@ -1002,6 +1008,42 @@ public void testNearAccess() throws Exception {
         checkTtl(key, 62_000L, true);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNearExpiresWithCacheStore() throws Exception {
+        factory = CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 1));
+
+        nearCache = true;
+
+        startGridsMultiThreaded(gridCount());
+
+        IgniteConfiguration clientCfg = getConfiguration("client").setClientMode(true);
+
+        ((TcpDiscoverySpi)clientCfg.getDiscoverySpi()).setForceServerMode(false);
+
+        Ignite client = startGrid("client", clientCfg);
+
+        CacheConfiguration ccfg = cacheConfiguration("testCache");
+
+//        ccfg.setExpiryPolicyFactory( CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 1)));
+
+        IgniteCache<Object, Object> cache = client.getOrCreateCache(ccfg);
+
+        Integer key = 1;
+
+        cache.put(key, 1);
+
+        assertEquals(1, cache.localPeek(key, CachePeekMode.NEAR));
+        assertEquals(1, cache.get(key));
+
+        waitExpired(key);
+
+        for(int i = 0; i < gridCount(); i++)
+            assertNull(jcache(i).localPeek(key, CachePeekMode.BACKUP, CachePeekMode.PRIMARY));
+
+        assertEquals(null, cache.get(key));
+    }
     /**
      * @return Test keys.
      * @throws Exception If failed.

From cf9f0a79da7540dc40c3ac860a94158e87e2d7ec Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 12 Jan 2017 17:50:02 +0300
Subject: [PATCH 007/516] Minors

---
 .../cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java        | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index 4368a1533f6d3..9739350d10617 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -1026,6 +1026,7 @@ public void testNearExpiresWithCacheStore() throws Exception {
 
         CacheConfiguration ccfg = cacheConfiguration("testCache");
 
+        ccfg.setCacheStoreFactory(FactoryBuilder.factoryOf(GridCacheTestStore.class));
 //        ccfg.setExpiryPolicyFactory( CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 1)));
 
         IgniteCache<Object, Object> cache = client.getOrCreateCache(ccfg);

From 295f80d10da02692540435d46961a9c3cca99b3c Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 25 Jan 2017 13:58:57 +0700
Subject: [PATCH 008/516] IGNITE-4520 Added credential request for
 authentication on proxy.

(cherry picked from commit ef04f35)
---
 .../web-agent/bin/ignite-web-agent.bat        |  4 +-
 .../web-agent/bin/ignite-web-agent.sh         |  2 +
 .../ignite/console/agent/AgentLauncher.java   | 90 +++++++++++++++++--
 3 files changed, 88 insertions(+), 8 deletions(-)

diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.bat b/modules/web-console/web-agent/bin/ignite-web-agent.bat
index 8291b5531da50..1f1b52dc83d23 100644
--- a/modules/web-console/web-agent/bin/ignite-web-agent.bat
+++ b/modules/web-console/web-agent/bin/ignite-web-agent.bat
@@ -60,7 +60,9 @@ if %ERRORLEVEL% equ 0 (
     if "%JVM_OPTS%" == "" set JVM_OPTS=-Xms1g -Xmx1g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=256m
 )
 
-"%JAVA_HOME%\bin\java.exe" %JVM_OPTS% -cp "*" org.apache.ignite.console.agent.AgentLauncher  %*
+set JVM_OPTS=%JVM_OPTS% -Djava.net.useSystemProxies=true
+
+"%JAVA_HOME%\bin\java.exe" %JVM_OPTS% -cp "*" org.apache.ignite.console.agent.AgentLauncher %*
 
 set JAVA_ERRORLEVEL=%ERRORLEVEL%
 
diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.sh b/modules/web-console/web-agent/bin/ignite-web-agent.sh
index 2e9f041f8ca7c..c2958fc0c7c9a 100644
--- a/modules/web-console/web-agent/bin/ignite-web-agent.sh
+++ b/modules/web-console/web-agent/bin/ignite-web-agent.sh
@@ -88,4 +88,6 @@ if [ -z "$JVM_OPTS" ] ; then
     fi
 fi
 
+JVM_OPTS="${JVM_OPTS} -Djava.net.useSystemProxies=true"
+
 "$JAVA" ${JVM_OPTS} -cp "*" org.apache.ignite.console.agent.AgentLauncher "$@"
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 0c03d7787b15d..049791f9107dc 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -25,11 +25,15 @@
 import io.socket.emitter.Emitter;
 import java.io.File;
 import java.io.IOException;
+import java.net.Authenticator;
 import java.net.ConnectException;
+import java.net.PasswordAuthentication;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.UnknownHostException;
 import java.util.Arrays;
+import java.util.Scanner;
 import java.util.concurrent.CountDownLatch;
 import java.util.jar.Attributes;
 import java.util.jar.Manifest;
@@ -76,9 +80,6 @@ public class AgentLauncher {
     /** */
     private static final String EVENT_AGENT_CLOSE = "agent:close";
 
-    /** */
-    private static final int RECONNECT_INTERVAL = 3000;
-
     /**
      * Create a trust manager that trusts all certificates It is not using a particular keyStore
      */
@@ -121,6 +122,15 @@ public void checkServerTrusted(
                     System.exit(1);
                 }
 
+                ignore = X.cause(e, UnknownHostException.class);
+
+                if (ignore != null) {
+                    log.error("Failed to establish connection to server, due to errors with DNS or missing proxy settings.");
+                    log.error("Documentation for proxy configuration can be found here: http://apacheignite.readme.io/docs/web-agent#section-proxy-configuration");
+
+                    System.exit(1);
+                }
+
                 ignore = X.cause(e, IOException.class);
 
                 if (ignore != null && "404".equals(ignore.getMessage())) {
@@ -129,6 +139,29 @@ public void checkServerTrusted(
                     return;
                 }
 
+                if (ignore != null && "407".equals(ignore.getMessage())) {
+                    log.error("Failed to establish connection to server, due to proxy requires authentication.");
+
+                    String userName = System.getProperty("https.proxyUsername", System.getProperty("http.proxyUsername"));
+
+                    if (userName == null || userName.trim().isEmpty())
+                        userName = readLine("Enter proxy user name: ");
+                    else
+                        System.out.println("Read username from system properties: " + userName);
+
+                    char[] pwd = readPassword("Enter proxy password: ");
+
+                    final PasswordAuthentication pwdAuth = new PasswordAuthentication(userName, pwd);
+
+                    Authenticator.setDefault(new Authenticator() {
+                        @Override protected PasswordAuthentication getPasswordAuthentication() {
+                            return pwdAuth;
+                        }
+                    });
+
+                    return;
+                }
+
                 log.error("Connection error.", e);
             }
         }
@@ -143,6 +176,32 @@ public void checkServerTrusted(
         }
     };
 
+    /**
+     * @param fmt Format string.
+     * @param args Arguments.
+     */
+    private static String readLine(String fmt, Object ... args) {
+        if (System.console() != null)
+            return System.console().readLine(fmt, args);
+
+        System.out.print(String.format(fmt, args));
+
+        return new Scanner(System.in).nextLine();
+    }
+
+    /**
+     * @param fmt Format string.
+     * @param args Arguments.
+     */
+    private static char[] readPassword(String fmt, Object ... args) {
+        if (System.console() != null)
+            return System.console().readPassword(fmt, args);
+
+        System.out.print(String.format(fmt, args));
+
+        return new Scanner(System.in).nextLine().toCharArray();
+    }
+
     /**
      * @param args Args.
      */
@@ -214,9 +273,9 @@ public static void main(String[] args) throws Exception {
             System.out.println("Security token is required to establish connection to the web console.");
             System.out.println(String.format("It is available on the Profile page: https://%s/profile", webHost));
 
-            System.out.print("Enter security tokens separated by comma: ");
+            String tokens = String.valueOf(readPassword("Enter security tokens separated by comma: "));
 
-            cfg.tokens(Arrays.asList(System.console().readLine().trim().split(",")));
+            cfg.tokens(Arrays.asList(tokens.trim().split(",")));
         }
 
         final RestHandler restHnd = new RestHandler(cfg);
@@ -226,12 +285,29 @@ public static void main(String[] args) throws Exception {
 
             URI uri = URI.create(cfg.serverUri());
 
+            // Create proxy authenticator using passed properties.
+            switch (uri.getScheme()) {
+                case "http":
+                case "https":
+                    final String username = System.getProperty(uri.getScheme() + ".proxyUsername");
+                    final char[] pwd = System.getProperty(uri.getScheme() +  ".proxyPassword", "").toCharArray();
+
+                    Authenticator.setDefault(new Authenticator() {
+                        @Override protected PasswordAuthentication getPasswordAuthentication() {
+                            return new PasswordAuthentication(username, pwd);
+                        }
+                    });
+
+                    break;
+
+                default:
+                    // No-op.
+            }
+
             IO.Options opts = new IO.Options();
 
             opts.path = "/agents";
 
-            opts.reconnectionDelay = RECONNECT_INTERVAL;
-
             // Workaround for use self-signed certificate
             if (Boolean.getBoolean("trust.all")) {
                 SSLContext ctx = SSLContext.getInstance("TLS");

From faa20530a707014ac34477cba8adaaa4b67de1bb Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 25 Jan 2017 16:48:05 +0700
Subject: [PATCH 009/516] IGNITE-1596 Fixed version sort.

(cherry picked from commit 128ba07)
---
 modules/web-console/backend/app/agent.js | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index 791ea50ed58b9..961253f148a70 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -650,14 +650,14 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo)
                         const bParts = b.split('.');
 
                         for (let i = 0; i < aParts.length; ++i) {
-                            if (bParts.length === i)
-                                return 1;
+                            if (aParts[i] !== bParts[i])
+                                return aParts[i] < bParts[i] ? 1 : -1;
+                        }
 
-                            if (aParts[i] === aParts[i])
-                                continue;
+                        if (aParts.length === bParts.length)
+                            return 0;
 
-                            return aParts[i] > bParts[i] ? 1 : -1;
-                        }
+                        return aParts.length < bParts.length ? 1 : -1;
                     }));
 
                     // Latest version of agent distribution.

From 1b3bbdb1758eb19c755dabcb5fe329529fa0f378 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 27 Jan 2017 11:30:49 +0700
Subject: [PATCH 010/516] IGNITE-4622 Fixed generation in domain model for
 cache store.

(cherry picked from commit 43007d5)
---
 .../configuration/generator/ConfigurationGenerator.js    | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index de2b750675fd8..670b8284f14c1 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -48,6 +48,10 @@ export default class IgniteConfigurationGenerator {
         return new Bean('org.apache.ignite.cache.QueryEntity', 'qryEntity', domain, cacheDflts);
     }
 
+    static domainJdbcTypeBean(domain) {
+        return new Bean('org.apache.ignite.cache.store.jdbc.JdbcType', 'type', domain);
+    }
+
     static discoveryConfigurationBean(discovery) {
         return new Bean('org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi', 'discovery', discovery, clusterDflts.discovery);
     }
@@ -1383,7 +1387,7 @@ export default class IgniteConfigurationGenerator {
     }
 
     // Generate domain model for store group.
-    static domainStore(domain, cfg = this.domainConfigurationBean(domain)) {
+    static domainStore(domain, cfg = this.domainJdbcTypeBean(domain)) {
         cfg.stringProperty('databaseSchema')
             .stringProperty('databaseTable');
 
@@ -1562,8 +1566,7 @@ export default class IgniteConfigurationGenerator {
                         if (_.isNil(domain.databaseTable))
                             return acc;
 
-                        const typeBean = new Bean('org.apache.ignite.cache.store.jdbc.JdbcType', 'type',
-                            _.merge({}, domain, {cacheName: cache.name}))
+                        const typeBean = this.domainJdbcTypeBean(_.merge({}, domain, {cacheName: cache.name}))
                             .stringProperty('cacheName');
 
                         setType(typeBean, 'keyType');

From a8355feef630a5c8c4edbe02224c8325dd5952d3 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 3 Feb 2017 11:58:43 +0700
Subject: [PATCH 011/516] IGNITE-4610 Added more informative message.

(cherry picked from commit 9ff2a8f)
---
 .../web-console/frontend/views/templates/agent-download.jade    | 2 +-
 modules/web-console/web-agent/README.txt                        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/frontend/views/templates/agent-download.jade b/modules/web-console/frontend/views/templates/agent-download.jade
index 20b6b03d058f3..f57bf1d87e21c 100644
--- a/modules/web-console/frontend/views/templates/agent-download.jade
+++ b/modules/web-console/frontend/views/templates/agent-download.jade
@@ -41,7 +41,7 @@
                 p Connection to Ignite Web Agent is established, but agent failed to connect to Ignite Node
                 p Please check the following:
                 ul
-                    li Ignite Grid is up and running
+                    li Ignite Grid is up and Ignite REST server started (copy "ignite-rest-http" folder from libs/optional/ to libs/)
                     li In agent settings check URI for connect to Ignite REST server
                     li Check agent logs for errors
                     li Refer to #[b README.txt] in agent folder for more information
diff --git a/modules/web-console/web-agent/README.txt b/modules/web-console/web-agent/README.txt
index cc0c80fc99adb..81df1cb4f0234 100644
--- a/modules/web-console/web-agent/README.txt
+++ b/modules/web-console/web-agent/README.txt
@@ -32,7 +32,7 @@ Security tokens:
   3) One may specify several comma separated tokens using configuration file or command line arguments of web agent.
 
 Ignite Web agent requirements:
-  1) In order to communicate with web agent Ignite node should be started with REST server (move ignite-rest-http folder from lib/optional/ to lib/).
+  1) In order to communicate with web agent Ignite node should be started with REST server (copy "ignite-rest-http" folder from "libs/optional/" to "libs/").
   2) Configure web agent serverURI property by Ignite node REST server URI.
 
 Options:

From 94d8a6fc0a61bb3046e2ebd8b3cbffb8917c2b6a Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 8 Feb 2017 11:43:22 +0700
Subject: [PATCH 012/516] IGNITE-4472 Added user activities in Web Console.

(cherry picked from commit 26ee9c2)
---
 modules/web-console/backend/app/agent.js      |  10 +-
 modules/web-console/backend/app/mongo.js      |  49 ++--
 modules/web-console/backend/app/routes.js     |   5 +-
 .../web-console/backend/routes/activities.js  |  52 ++++
 modules/web-console/backend/routes/admin.js   |   2 +-
 modules/web-console/backend/routes/agent.js   |  10 +-
 modules/web-console/backend/routes/public.js  |   1 -
 .../backend/services/activities.js            | 136 ++++++++++
 modules/web-console/backend/services/users.js |  13 +-
 .../web-console/frontend/app/app.config.js    |   9 +
 modules/web-console/frontend/app/app.js       |  29 ++-
 .../activities-user-dialog.controller.js      |  60 +++++
 .../activities-user-dialog.jade               |  36 +++
 .../activities-user-dialog/index.js           |  36 +++
 .../form-field-datepicker.jade                |  55 ++++
 .../form-field-datepicker.scss                |  20 ++
 .../list-of-registered-users/index.js         |  28 +++
 .../list-of-registered-users.categories.js    |  30 +++
 .../list-of-registered-users.column-defs.js   |  80 ++++++
 .../list-of-registered-users.controller.js    | 207 ++++++++++++++++
 .../list-of-registered-users.jade             |  54 ++++
 .../ui-grid-header/ui-grid-header.jade        |  27 ++
 .../ui-grid-header/ui-grid-header.scss        |  84 +++++++
 .../ui-grid-settings/ui-grid-settings.jade    |   0
 .../ui-grid-settings/ui-grid-settings.scss    |  32 +++
 .../app/core/activities/Activities.data.js    |  39 +++
 .../frontend/app/core/admin/Admin.data.js     |  77 ++++++
 .../web-console/frontend/app/core/index.js    |  25 ++
 modules/web-console/frontend/app/data/i18n.js |  38 +++
 .../app/filters/uiGridSubcategories.filter.js |  24 ++
 .../app/modules/sql/sql.controller.js         |  14 +-
 .../frontend/app/modules/sql/sql.module.js    |   2 +-
 .../app/modules/states/admin.state.js         |   2 +-
 .../summary/summary.controller.js             |   6 +-
 .../app/modules/user/AclRoute.provider.js     |  31 ++-
 .../frontend/app/modules/user/Auth.service.js |   2 +-
 .../frontend/app/modules/user/permissions.js  |   2 +-
 .../frontend/app/modules/user/user.module.js  |   6 +-
 modules/web-console/frontend/app/vendor.js    |   1 +
 .../frontend/controllers/admin-controller.js  | 234 ------------------
 .../controllers/domains-controller.js         |  12 +-
 modules/web-console/frontend/package.json     |   1 +
 .../stylesheets/_font-awesome-custom.scss     |  28 +++
 .../frontend/public/stylesheets/style.scss    |  39 ++-
 .../public/stylesheets/variables.scss         |   1 +
 .../frontend/views/settings/admin.jade        |  32 +--
 .../web-console/frontend/views/sql/sql.jade   |   4 +-
 47 files changed, 1360 insertions(+), 325 deletions(-)
 create mode 100644 modules/web-console/backend/routes/activities.js
 create mode 100644 modules/web-console/backend/services/activities.js
 create mode 100644 modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
 create mode 100644 modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
 create mode 100644 modules/web-console/frontend/app/components/activities-user-dialog/index.js
 create mode 100644 modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
 create mode 100644 modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.scss
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/index.js
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.categories.js
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
 create mode 100644 modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
 create mode 100644 modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
 rename modules/web-console/frontend/app/{directives => components}/ui-grid-settings/ui-grid-settings.jade (100%)
 rename modules/web-console/frontend/app/{directives => components}/ui-grid-settings/ui-grid-settings.scss (68%)
 create mode 100644 modules/web-console/frontend/app/core/activities/Activities.data.js
 create mode 100644 modules/web-console/frontend/app/core/admin/Admin.data.js
 create mode 100644 modules/web-console/frontend/app/core/index.js
 create mode 100644 modules/web-console/frontend/app/data/i18n.js
 create mode 100644 modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js
 delete mode 100644 modules/web-console/frontend/controllers/admin-controller.js

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index 961253f148a70..817028059f50f 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -24,7 +24,7 @@
  */
 module.exports = {
     implements: 'agent-manager',
-    inject: ['require(lodash)', 'require(fs)', 'require(path)', 'require(jszip)', 'require(socket.io)', 'settings', 'mongo']
+    inject: ['require(lodash)', 'require(fs)', 'require(path)', 'require(jszip)', 'require(socket.io)', 'settings', 'mongo', 'services/activities']
 };
 
 /**
@@ -35,9 +35,10 @@ module.exports = {
  * @param socketio
  * @param settings
  * @param mongo
+ * @param {ActivitiesService} activitiesService
  * @returns {AgentManager}
  */
-module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo) {
+module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo, activitiesService) {
     /**
      *
      */
@@ -823,6 +824,11 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo)
                 const sockets = this._browsers[accountId];
 
                 _.forEach(sockets, (socket) => socket.emit('agent:count', {count: agents.length}));
+
+                activitiesService.merge(accountId, {
+                    group: 'agent',
+                    action: '/agent/start'
+                });
             });
         }
 
diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index dd71f3a8bf57e..2d252b9b8eccc 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -48,6 +48,7 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
         company: String,
         country: String,
         lastLogin: Date,
+        lastActivity: Date,
         admin: Boolean,
         token: String,
         resetPasswordToken: String
@@ -59,22 +60,26 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
         usernameLowerCase: true
     });
 
+    const transform = (doc, ret) => {
+        return {
+            _id: ret._id,
+            email: ret.email,
+            firstName: ret.firstName,
+            lastName: ret.lastName,
+            company: ret.company,
+            country: ret.country,
+            admin: ret.admin,
+            token: ret.token,
+            lastLogin: ret.lastLogin,
+            lastActivity: ret.lastActivity
+        };
+    };
+
     // Configure transformation to JSON.
-    AccountSchema.set('toJSON', {
-        transform: (doc, ret) => {
-            return {
-                _id: ret._id,
-                email: ret.email,
-                firstName: ret.firstName,
-                lastName: ret.lastName,
-                company: ret.company,
-                country: ret.country,
-                admin: ret.admin,
-                token: ret.token,
-                lastLogin: ret.lastLogin
-            };
-        }
-    });
+    AccountSchema.set('toJSON', { transform });
+
+    // Configure transformation to JSON.
+    AccountSchema.set('toObject', { transform });
 
     result.errCodes = {
         DUPLICATE_KEY_ERROR: 11000,
@@ -902,6 +907,20 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
         res.status(err.code || 500).send(err.message);
     };
 
+    // Define Activities schema.
+    const ActivitiesSchema = new Schema({
+        owner: {type: ObjectId, ref: 'Account'},
+        date: Date,
+        group: String,
+        action: String,
+        amount: { type: Number, default: 1 }
+    });
+
+    ActivitiesSchema.index({ owner: 1, group: 1, action: 1, date: 1}, { unique: true });
+
+    // Define Activities model.
+    result.Activities = mongoose.model('Activities', ActivitiesSchema);
+
     // Registering the routes of all plugin modules
     for (const name in pluginMongo) {
         if (pluginMongo.hasOwnProperty(name))
diff --git a/modules/web-console/backend/app/routes.js b/modules/web-console/backend/app/routes.js
index 6961173d03eb9..6b5d052d4d765 100644
--- a/modules/web-console/backend/app/routes.js
+++ b/modules/web-console/backend/app/routes.js
@@ -22,11 +22,11 @@
 module.exports = {
     implements: 'routes',
     inject: ['routes/public', 'routes/admin', 'routes/profiles', 'routes/demo', 'routes/clusters', 'routes/domains',
-        'routes/caches', 'routes/igfss', 'routes/notebooks', 'routes/agents', 'routes/configurations']
+        'routes/caches', 'routes/igfss', 'routes/notebooks', 'routes/agents', 'routes/configurations', 'routes/activities']
 };
 
 module.exports.factory = function(publicRoute, adminRoute, profilesRoute, demoRoute,
-    clustersRoute, domainsRoute, cachesRoute, igfssRoute, notebooksRoute, agentsRoute, configurationsRoute) {
+    clustersRoute, domainsRoute, cachesRoute, igfssRoute, notebooksRoute, agentsRoute, configurationsRoute, activitiesRoute) {
     return {
         register: (app) => {
             const _mustAuthenticated = (req, res, next) => {
@@ -59,6 +59,7 @@ module.exports.factory = function(publicRoute, adminRoute, profilesRoute, demoRo
 
             app.use('/notebooks', _mustAuthenticated, notebooksRoute);
             app.use('/agent', _mustAuthenticated, agentsRoute);
+            app.use('/activities', _mustAuthenticated, activitiesRoute);
         }
     };
 };
diff --git a/modules/web-console/backend/routes/activities.js b/modules/web-console/backend/routes/activities.js
new file mode 100644
index 0000000000000..08c27cf1cdf8c
--- /dev/null
+++ b/modules/web-console/backend/routes/activities.js
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+// Fire me up!
+
+module.exports = {
+    implements: 'routes/activities',
+    inject: ['require(express)', 'services/activities']
+};
+
+/**
+ * @param express
+ * @param {ActivitiesService} activitiesService
+ * @returns {Promise}
+ */
+module.exports.factory = function(express, activitiesService) {
+    return new Promise((factoryResolve) => {
+        const router = new express.Router();
+
+        // Get user activities.
+        router.get('/user/:userId', (req, res) => {
+            activitiesService.listByUser(req.params.userId, req.query)
+                .then(res.api.ok)
+                .catch(res.api.error);
+        });
+
+        // Post user activities to page.
+        router.post('/page', (req, res) => {
+            activitiesService.merge(req.user._id, req.body)
+                .then(res.api.ok)
+                .catch(res.api.error);
+        });
+
+        factoryResolve(router);
+    });
+};
diff --git a/modules/web-console/backend/routes/admin.js b/modules/web-console/backend/routes/admin.js
index 70736d00ea43a..5b0896ac69437 100644
--- a/modules/web-console/backend/routes/admin.js
+++ b/modules/web-console/backend/routes/admin.js
@@ -43,7 +43,7 @@ module.exports.factory = function(_, express, settings, mongo, spacesService, ma
          * Get list of user accounts.
          */
         router.post('/list', (req, res) => {
-            usersService.list()
+            usersService.list(req.body)
                 .then(res.api.ok)
                 .catch(res.api.error);
         });
diff --git a/modules/web-console/backend/routes/agent.js b/modules/web-console/backend/routes/agent.js
index 477363f9f5266..5ae807b5e88c1 100644
--- a/modules/web-console/backend/routes/agent.js
+++ b/modules/web-console/backend/routes/agent.js
@@ -21,21 +21,27 @@
 
 module.exports = {
     implements: 'routes/agents',
-    inject: ['require(lodash)', 'require(express)', 'services/agents']
+    inject: ['require(lodash)', 'require(express)', 'services/agents', 'services/activities']
 };
 
 /**
  * @param _
  * @param express
  * @param {AgentsService} agentsService
+ * @param {ActivitiesService} activitiesService
  * @returns {Promise}
  */
-module.exports.factory = function(_, express, agentsService) {
+module.exports.factory = function(_, express, agentsService, activitiesService) {
     return new Promise((resolveFactory) => {
         const router = new express.Router();
 
         /* Get grid topology. */
         router.get('/download/zip', (req, res) => {
+            activitiesService.merge(req.user._id, {
+                group: 'agent',
+                action: '/agent/download'
+            });
+
             agentsService.getArchive(req.origin(), req.user.token)
                 .then(({fileName, buffer}) => {
                     // Set the archive name.
diff --git a/modules/web-console/backend/routes/public.js b/modules/web-console/backend/routes/public.js
index 590d395179abd..860e2678c2fff 100644
--- a/modules/web-console/backend/routes/public.js
+++ b/modules/web-console/backend/routes/public.js
@@ -25,7 +25,6 @@ module.exports = {
 };
 
 /**
- *
  * @param express
  * @param passport
  * @param mongo
diff --git a/modules/web-console/backend/services/activities.js b/modules/web-console/backend/services/activities.js
new file mode 100644
index 0000000000000..7f3a7775f83d2
--- /dev/null
+++ b/modules/web-console/backend/services/activities.js
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+// Fire me up!
+
+module.exports = {
+    implements: 'services/activities',
+    inject: ['require(lodash)', 'mongo']
+};
+
+/**
+ * @param _
+ * @param mongo
+ * @returns {ActivitiesService}
+ */
+module.exports.factory = (_, mongo) => {
+    class ActivitiesService {
+        /**
+         * Update page activities.
+         *
+         * @param {String} owner - User ID
+         * @param {Object} page - The page
+         * @returns {Promise.<mongo.ObjectId>} that resolve activity
+         */
+        static merge(owner, {action, group}) {
+            mongo.Account.findById(owner)
+                .then((user) => {
+                    user.lastActivity = new Date();
+
+                    return user.save();
+                });
+
+            const date = new Date();
+
+            date.setDate(1);
+            date.setHours(0, 0, 0, 0);
+
+            return mongo.Activities.findOne({owner, action, date}).exec()
+                .then((activity) => {
+                    if (activity) {
+                        activity.amount++;
+
+                        return activity.save();
+                    }
+
+                    return mongo.Activities.create({owner, action, group, date});
+                });
+        }
+
+        /**
+         * Get user activities
+         * @param {String} owner - User ID
+         * @returns {Promise.<mongo.ObjectId>} that resolve activities
+         */
+        static listByUser(owner, {startDate, endDate}) {
+            const $match = {owner};
+
+            if (startDate)
+                $match.date = {$gte: new Date(startDate)};
+
+            if (endDate) {
+                $match.date = $match.date || {};
+                $match.date.$lt = new Date(endDate);
+            }
+
+            return mongo.Activities.find($match);
+        }
+
+        static total({startDate, endDate}) {
+            const $match = {};
+
+            if (startDate)
+                $match.date = {$gte: new Date(startDate)};
+
+            if (endDate) {
+                $match.date = $match.date || {};
+                $match.date.$lt = new Date(endDate);
+            }
+
+            return mongo.Activities.aggregate([
+                {$match},
+                {$group: {
+                    _id: {owner: '$owner', group: '$group'},
+                    amount: {$sum: '$amount'}
+                }}
+            ]).exec().then((data) => {
+                return _.reduce(data, (acc, { _id, amount }) => {
+                    const {owner, group} = _id;
+                    acc[owner] = _.merge(acc[owner] || {}, { [group]: amount });
+                    return acc;
+                }, {});
+            });
+        }
+
+        static detail({startDate, endDate}) {
+            const $match = { };
+
+            if (startDate)
+                $match.date = {$gte: new Date(startDate)};
+
+            if (endDate) {
+                $match.date = $match.date || {};
+                $match.date.$lt = new Date(endDate);
+            }
+
+            return mongo.Activities.aggregate([
+                {$match},
+                {$group: {_id: {owner: '$owner', action: '$action'}, total: {$sum: '$amount'}}}
+            ]).exec().then((data) => {
+                return _.reduce(data, (acc, { _id, total }) => {
+                    const {owner, action} = _id;
+                    acc[owner] = _.merge(acc[owner] || {}, { [action]: total });
+                    return acc;
+                }, {});
+            });
+        }
+    }
+
+    return ActivitiesService;
+};
diff --git a/modules/web-console/backend/services/users.js b/modules/web-console/backend/services/users.js
index 8058b25f9fed6..2dd603f193382 100644
--- a/modules/web-console/backend/services/users.js
+++ b/modules/web-console/backend/services/users.js
@@ -21,7 +21,7 @@
 
 module.exports = {
     implements: 'services/users',
-    inject: ['require(lodash)', 'mongo', 'settings', 'services/spaces', 'services/mails', 'agent-manager', 'errors']
+    inject: ['require(lodash)', 'mongo', 'settings', 'services/spaces', 'services/mails', 'services/activities', 'agent-manager', 'errors']
 };
 
 /**
@@ -30,11 +30,12 @@ module.exports = {
  * @param settings
  * @param {SpacesService} spacesService
  * @param {MailsService} mailsService
+ * @param {ActivitiesService} activitiesService
  * @param agentMgr
  * @param errors
  * @returns {UsersService}
  */
-module.exports.factory = (_, mongo, settings, spacesService, mailsService, agentMgr, errors) => {
+module.exports.factory = (_, mongo, settings, spacesService, mailsService, activitiesService, agentMgr, errors) => {
     const _randomString = () => {
         const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
         const possibleLen = possible.length;
@@ -143,7 +144,7 @@ module.exports.factory = (_, mongo, settings, spacesService, mailsService, agent
          * Get list of user accounts and summary information.
          * @returns {mongo.Account[]} - returns all accounts with counters object
          */
-        static list() {
+        static list(params) {
             return Promise.all([
                 mongo.Space.aggregate([
                     {$match: {demo: false}},
@@ -161,13 +162,17 @@ module.exports.factory = (_, mongo, settings, spacesService, mailsService, agent
                         }
                     }
                 ]).exec(),
+                activitiesService.total(params),
+                activitiesService.detail(params),
                 mongo.Account.find({}).sort('firstName lastName').lean().exec()
             ])
-                .then(([counters, users]) => {
+                .then(([counters, activitiesTotal, activitiesDetail, users]) => {
                     const countersMap = _.keyBy(counters, 'owner');
 
                     _.forEach(users, (user) => {
                         user.counters = _.omit(countersMap[user._id], '_id', 'owner');
+                        user.activitiesTotal = activitiesTotal[user._id];
+                        user.activitiesDetail = activitiesDetail[user._id];
                     });
 
                     return users;
diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js
index 7416ce9e3a283..0e85711141799 100644
--- a/modules/web-console/frontend/app/app.config.js
+++ b/modules/web-console/frontend/app/app.config.js
@@ -94,3 +94,12 @@ igniteConsoleCfg.config(['$dropdownProvider', ($dropdownProvider) => {
         templateUrl: 'templates/dropdown.html'
     });
 }]);
+
+// AngularStrap dropdowns () configuration.
+igniteConsoleCfg.config(['$datepickerProvider', ($datepickerProvider) => {
+    angular.extend($datepickerProvider.defaults, {
+        autoclose: true,
+        iconLeft: 'icon-datepicker-left',
+        iconRight: 'icon-datepicker-right'
+    });
+}]);
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 4ecd9b5394483..9958cb53ae539 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -16,7 +16,9 @@
  */
 
 import '../public/stylesheets/style.scss';
-import '../app/directives/ui-grid-settings/ui-grid-settings.scss';
+import '../app/components/ui-grid-header/ui-grid-header.scss';
+import '../app/components/ui-grid-settings/ui-grid-settings.scss';
+import '../app/components/form-field-datepicker/form-field-datepicker.scss';
 import './helpers/jade/mixins.jade';
 
 import './app.config';
@@ -25,10 +27,10 @@ import './decorator/select';
 import './decorator/tooltip';
 
 import './modules/form/form.module';
-import './modules/agent/agent.module.js';
+import './modules/agent/agent.module';
 import './modules/sql/sql.module';
 import './modules/nodes/nodes.module';
-import './modules/Demo/Demo.module.js';
+import './modules/demo/Demo.module';
 
 import './modules/states/signin.state';
 import './modules/states/logout.state';
@@ -39,6 +41,7 @@ import './modules/states/admin.state';
 import './modules/states/errors.state';
 
 // ignite:modules
+import './core';
 import './modules/user/user.module';
 import './modules/branding/branding.module';
 import './modules/navbar/navbar.module';
@@ -50,6 +53,9 @@ import './modules/socket.module';
 import './modules/loading/loading.module';
 // endignite
 
+// Data
+import i18n from './data/i18n';
+
 // Directives.
 import igniteAutoFocus from './directives/auto-focus.directive.js';
 import igniteBsAffixUpdate from './directives/bs-affix-update.directive';
@@ -98,9 +104,9 @@ import defaultName from './filters/default-name.filter';
 import domainsValidation from './filters/domainsValidation.filter';
 import duration from './filters/duration.filter';
 import hasPojo from './filters/hasPojo.filter';
+import uiGridSubcategories from './filters/uiGridSubcategories.filter';
 
 // Controllers
-import admin from 'controllers/admin-controller';
 import caches from 'controllers/caches-controller';
 import clusters from 'controllers/clusters-controller';
 import domains from 'controllers/domains-controller';
@@ -109,6 +115,10 @@ import profile from 'controllers/profile-controller';
 import auth from './controllers/auth.controller';
 import resetPassword from './controllers/reset-password.controller';
 
+// Components
+import igniteListOfRegisteredUsers from './components/list-of-registered-users';
+import IgniteActivitiesUserDialog from './components/activities-user-dialog';
+
 // Inject external modules.
 import 'ignite_modules_temp/index';
 
@@ -129,6 +139,7 @@ angular
     'nvd3',
     'smart-table',
     'treeControl',
+    'pascalprecht.translate',
     'ui.grid',
     'ui.grid.saveState',
     'ui.grid.selection',
@@ -136,6 +147,7 @@ angular
     'ui.grid.autoResize',
     'ui.grid.exporter',
     // Base modules.
+    'ignite-console.core',
     'ignite-console.ace',
     'ignite-console.Form',
     'ignite-console.user',
@@ -186,6 +198,7 @@ angular
 .directive(...igniteRetainSelection)
 .directive('igniteOnFocusOut', igniteOnFocusOut)
 .directive('igniteRestoreInputFocus', igniteRestoreInputFocus)
+.directive('igniteListOfRegisteredUsers', igniteListOfRegisteredUsers)
 // Services.
 .service('IgniteErrorPopover', ErrorPopover)
 .service('JavaTypes', JavaTypes)
@@ -204,8 +217,8 @@ angular
 .service(...FormUtils)
 .service(...LegacyUtils)
 .service(...UnsavedChangesGuard)
+.service('IgniteActivitiesUserDialog', IgniteActivitiesUserDialog)
 // Controllers.
-.controller(...admin)
 .controller(...auth)
 .controller(...resetPassword)
 .controller(...caches)
@@ -219,7 +232,11 @@ angular
 .filter(...domainsValidation)
 .filter(...duration)
 .filter(...hasPojo)
-.config(['$stateProvider', '$locationProvider', '$urlRouterProvider', ($stateProvider, $locationProvider, $urlRouterProvider) => {
+.filter('uiGridSubcategories', uiGridSubcategories)
+.config(['$translateProvider', '$stateProvider', '$locationProvider', '$urlRouterProvider', ($translateProvider, $stateProvider, $locationProvider, $urlRouterProvider) => {
+    $translateProvider.translations('en', i18n);
+    $translateProvider.preferredLanguage('en');
+
     // Set up the states.
     $stateProvider
         .state('base', {
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
new file mode 100644
index 0000000000000..46853b262eaad
--- /dev/null
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const COLUMNS_DEFS = [
+    {displayName: 'Action', field: 'action', minWidth: 65 },
+    {displayName: 'Description', field: 'title', minWidth: 65 },
+    {displayName: 'Visited', field: 'amount', minWidth: 65 }
+];
+
+export default class ActivitiesCtrl {
+    static $inject = ['$state', 'user', 'params', 'IgniteActivitiesData'];
+
+    constructor($state, user, params, ActivitiesData) {
+        const $ctrl = this;
+        const userId = user._id;
+
+        $ctrl.user = user;
+
+        $ctrl.gridOptions = {
+            data: [],
+            columnVirtualizationThreshold: 30,
+            columnDefs: COLUMNS_DEFS,
+            categories: [
+                {name: 'Action', visible: true, selectable: true},
+                {name: 'Description', visible: true, selectable: true},
+                {name: 'Visited', visible: true, selectable: true}
+            ],
+            enableRowSelection: false,
+            enableRowHeaderSelection: false,
+            enableColumnMenus: false,
+            multiSelect: false,
+            modifierKeysToMultiSelect: true,
+            noUnselect: true,
+            flatEntityAccess: true,
+            fastWatch: true,
+            onRegisterApi: (api) => {
+                $ctrl.gridApi = api;
+            }
+        };
+
+        ActivitiesData.listByUser(userId, params)
+            .then((data) => {
+                $ctrl.data = data;
+            });
+    }
+}
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
new file mode 100644
index 0000000000000..2c55ebd6da29a
--- /dev/null
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
@@ -0,0 +1,36 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+.modal(tabindex='-1' role='dialog')
+    .modal-dialog
+        .modal-content
+            .modal-header
+                h4.modal-title 
+                    i.fa.fa-info-circle
+                    | Activities: {{ ctrl.user.userName }}
+            .modal-body.modal-body-with-scroll(id='activities-user-dialog')
+                table.table.table-striped.table-bordered.table-hover(scrollable-container='#activities-user-dialog' st-table='displayedRows' st-safe-src='ctrl.data')
+                    thead
+                        th.text-center(st-sort='title') Description
+                        th.text-center(st-sort='action') Action
+                        th.text-center(st-sort='amount') Visited
+                    tbody
+                        tr(ng-repeat='row in displayedRows')
+                            td.text-left {{ row.action | translate }}
+                            td.text-left {{ row.action }}
+                            td.text-left {{ row.amount }}
+            .modal-footer
+                button.btn.btn-primary(id='confirm-btn-confirm' ng-click='$hide()') Close
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/index.js b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
new file mode 100644
index 0000000000000..03d35852b147a
--- /dev/null
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ import controller from './activities-user-dialog.controller';
+ import templateUrl from './activities-user-dialog.jade';
+
+ export default ['$modal', ($modal) => ({ show = true, user, params }) => {
+     const ActivitiesUserDialog = $modal({
+         templateUrl,
+         show,
+         resolve: {
+             user: () => user,
+             params: () => params
+         },
+         placement: 'center',
+         controller,
+         controllerAs: 'ctrl'
+     });
+
+     return ActivitiesUserDialog.$promise
+         .then(() => ActivitiesUserDialog);
+ }];
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
new file mode 100644
index 0000000000000..6792977b65831
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
@@ -0,0 +1,55 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+mixin ignite-form-field-datepicker(label, model, name, disabled, required, placeholder, tip)
+    mixin form-field-input()
+        input.form-control(
+            id='{{ #{name} }}Input'
+            name='{{ #{name} }}'
+
+            placeholder=placeholder
+            
+            data-ng-model=model
+
+            data-ng-required=required && '#{required}'
+            data-ng-disabled=disabled && '#{disabled}'
+
+            bs-datepicker
+            data-date-format='MMM yyyy' 
+            data-start-view='1' 
+            data-min-view='1' 
+            data-max-date='today'
+
+            data-container='body > .wrapper'
+
+            tabindex='0'
+
+            onkeydown="return false"
+
+            data-ignite-form-panel-field=''
+        )&attributes(attributes.attributes)
+
+    .ignite-form-field
+        +ignite-form-field__label(label, name, required)
+        .ignite-form-field__control
+            if tip
+                i.tipField.icon-help(bs-tooltip='' data-title=tip)
+
+            if block
+                block
+
+            .input-tip
+                +form-field-input(attributes=attributes)
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.scss b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.scss
new file mode 100644
index 0000000000000..0f6fe6eb62519
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.scss
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.datepicker.dropdown-menu tbody button {
+    height: 100%;
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/index.js b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
new file mode 100644
index 0000000000000..32a34f41c02d6
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import templateUrl from './list-of-registered-users.jade';
+import controller from './list-of-registered-users.controller';
+
+export default [() => {
+    return {
+        scope: true,
+        templateUrl,
+        controller,
+        controllerAs: '$ctrl'
+    };
+}];
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.categories.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.categories.js
new file mode 100644
index 0000000000000..95edf8b092b66
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.categories.js
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default [
+    {name: 'Actions', visible: true, selectable: true},
+    {name: 'User', visible: true, selectable: true},
+    {name: 'Email', visible: true, selectable: true},
+    {name: 'Company', visible: true, selectable: true},
+    {name: 'Country', visible: true, selectable: true},
+    {name: 'Last login', visible: false, selectable: true},
+    {name: 'Last activity', visible: true, selectable: true},
+    {name: 'Configurations', visible: false, selectable: true},
+    {name: 'Total activities', visible: true, selectable: true},
+    {name: 'Configuration\'s activities', visible: false, selectable: true},
+    {name: 'Queries\' activities', visible: false, selectable: true}
+];
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
new file mode 100644
index 0000000000000..61e1bd8575b22
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const ICON_SORT = '<span ui-grid-one-bind-id-grid="col.uid + \'-sortdir-text\'" ui-grid-visible="col.sort.direction" aria-label="Sort Descending"><i ng-class="{ \'ui-grid-icon-up-dir\': col.sort.direction == asc, \'ui-grid-icon-down-dir\': col.sort.direction == desc, \'ui-grid-icon-blank\': !col.sort.direction }" title="" aria-hidden="true"></i></span>';
+
+const USER_TEMPLATE = '<div class="ui-grid-cell-contents"><i class="pull-left" ng-class="row.entity.admin ? \'icon-admin\' : \'icon-user\'"></i>{{ COL_FIELD }}</div>';
+
+const CLUSTER_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-sitemap'></i>${ICON_SORT}</div>`;
+const MODEL_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-object-group'></i>${ICON_SORT}</div>`;
+const CACHE_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-database'></i>${ICON_SORT}</div>`;
+const IGFS_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-folder-o'></i>${ICON_SORT}</div>`;
+
+const ACTIONS_TEMPLATE = `
+<div class='text-center ui-grid-cell-actions'>
+    <a class='btn btn-default dropdown-toggle' bs-dropdown='' data-placement='bottom-right' data-container='.panel'>
+        <i class='fa fa-gear'></i>&nbsp;
+        <span class='caret'></span>
+    </a>
+    <ul class='dropdown-menu' role='menu'>
+        <li ng-show='row.entity._id != $root.user._id'>
+            <a ng-click='grid.api.becomeUser(row.entity)'>Become this user</a>
+        </li>
+        <li ng-show='row.entity._id != $root.user._id'>
+            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='row.entity.admin && row.entity._id !== $root.user._id'>Revoke admin</a>
+            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='!row.entity.admin && row.entity._id !== $root.user._id'>Grant admin</a>
+        </li>
+        <li ng-show='row.entity._id != $root.user._id'>
+            <a ng-click='grid.api.removeUser(row.entity)'>Remove user</a>
+        </li>
+        <li>
+            <a ng-click='grid.api.showActivities(row.entity)'>Activity detail</a>
+        </li>
+</div>`;
+
+const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
+
+export default [
+    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'test', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, pinnedLeft: true},
+    {displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by name...' }, pinnedLeft: true},
+    {displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
+    {displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 160, enableFiltering: true},
+    {displayName: 'Country', categoryDisplayName: 'Country', field: 'countryCode', minWidth: 80, enableFiltering: true},
+    {displayName: 'Last login', categoryDisplayName: 'Last login', field: 'lastLogin', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 105, width: 105, enableFiltering: false, visible: false},
+    {displayName: 'Last activity', categoryDisplayName: 'Last activity', field: 'lastActivity', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 105, width: 105, enableFiltering: false, visible: true, sort: { direction: 'desc', priority: 0 }},
+    // Configurations
+    {displayName: 'Clusters count', categoryDisplayName: 'Configurations', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: 'counters.clusters', type: 'number', headerTooltip: 'Clusters count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
+    {displayName: 'Models count', categoryDisplayName: 'Configurations', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: 'counters.models', type: 'number', headerTooltip: 'Models count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
+    {displayName: 'Caches count', categoryDisplayName: 'Configurations', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: 'counters.caches', type: 'number', headerTooltip: 'Caches count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
+    {displayName: 'IGFS count', categoryDisplayName: 'Configurations', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: 'counters.igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
+    // Activities Total
+    {displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Configuration', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Queries', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Demo', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'AD', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Agent Download', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'AS', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Agent Start', minWidth: 50, width: 50, enableFiltering: false},
+    // Activities Configuration
+    {displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration Clusters', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration Model', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', headerTooltip: 'Configuration Caches', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'IGFS', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/igfs"] || 0', type: 'number', headerTooltip: 'Configuration IGFS', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', headerTooltip: 'Configuration Summary', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    // Activities Queries
+    {displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', headerTooltip: 'Query execute', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', headerTooltip: 'Query explain', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', headerTooltip: 'Scan', minWidth: 50, width: 80, enableFiltering: false, visible: false}
+];
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
new file mode 100644
index 0000000000000..19f7921ac7f17
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import headerTemplate from 'app/components/ui-grid-header/ui-grid-header.jade';
+
+import columnDefs from './list-of-registered-users.column-defs';
+import categories from './list-of-registered-users.categories';
+
+export default class IgniteListOfRegisteredUsersCtrl {
+    static $inject = ['$scope', '$state', '$templateCache', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
+
+    constructor($scope, $state, $templateCache, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
+        const $ctrl = this;
+
+        const companySelectOptions = [];
+        const countrySelectOptions = [];
+
+        $ctrl.params = {
+            startDate: new Date()
+        };
+
+        $ctrl.params.startDate.setDate(1);
+        $ctrl.params.startDate.setHours(0, 0, 0, 0);
+
+        const columnCompany = _.find(columnDefs, { displayName: 'Company' });
+        const columnCountry = _.find(columnDefs, { displayName: 'Country' });
+
+        columnCompany.filter = {
+            selectOptions: companySelectOptions,
+            type: uiGridConstants.filter.SELECT,
+            condition: uiGridConstants.filter.EXACT
+        };
+
+        columnCountry.filter = {
+            selectOptions: countrySelectOptions,
+            type: uiGridConstants.filter.SELECT,
+            condition: uiGridConstants.filter.EXACT
+        };
+
+        const becomeUser = (user) => {
+            AdminData.becomeUser(user._id)
+                .then(() => User.load())
+                .then(() => $state.go('base.configuration.clusters'))
+                .then(() => NotebookData.load());
+        };
+
+        const removeUser = (user) => {
+            Confirm.confirm(`Are you sure you want to remove user: "${user.userName}"?`)
+                .then(() => AdminData.removeUser(user))
+                .then(() => {
+                    const i = _.findIndex($ctrl.gridOptions.data, (u) => u._id === user._id);
+
+                    if (i >= 0)
+                        $ctrl.gridOptions.data.splice(i, 1);
+                })
+                .then(() => $ctrl.adjustHeight($ctrl.gridOptions.data.length));
+        };
+
+        const toggleAdmin = (user) => {
+            if (user.adminChanging)
+                return;
+
+            user.adminChanging = true;
+
+            AdminData.toggleAdmin(user)
+                .then(() => user.admin = !user.admin)
+                .finally(() => user.adminChanging = false);
+        };
+
+        const showActivities = (user) => {
+            return new ActivitiesUserDialog({ user, params: $ctrl.params });
+        };
+
+        $ctrl.gridOptions = {
+            data: [],
+            columnVirtualizationThreshold: 30,
+            columnDefs,
+            categories,
+            headerTemplate: $templateCache.get(headerTemplate),
+            enableFiltering: true,
+            enableRowSelection: false,
+            enableRowHeaderSelection: false,
+            enableColumnMenus: false,
+            multiSelect: false,
+            modifierKeysToMultiSelect: true,
+            noUnselect: true,
+            fastWatch: true,
+            onRegisterApi: (api) => {
+                $ctrl.gridApi = api;
+
+                api.becomeUser = becomeUser;
+                api.removeUser = removeUser;
+                api.toggleAdmin = toggleAdmin;
+                api.showActivities = showActivities;
+            }
+        };
+
+        const usersToFilterOptions = (column) => {
+            return _.sortBy(
+                _.map(
+                    _.groupBy($ctrl.gridOptions.data, (usr) => {
+                        const fld = usr[column];
+
+                        return _.isNil(fld) ? fld : fld.toUpperCase();
+                    }),
+                    (arr, value) => ({label: `${_.head(arr)[column] || 'Not set'} (${arr.length})`, value})
+                ),
+                'value');
+        };
+
+        /**
+         * @param {{startDate: Date, endDate: Date}} params
+         */
+        const reloadUsers = (params) => {
+            AdminData.loadUsers(params)
+                .then((data) => $ctrl.gridOptions.data = data)
+                .then((data) => {
+                    companySelectOptions.push(...usersToFilterOptions('company'));
+                    countrySelectOptions.push(...usersToFilterOptions('countryCode'));
+
+                    this.gridApi.grid.refresh();
+
+                    return data;
+                })
+                .then((data) => $ctrl.adjustHeight(data.length));
+        };
+
+        $scope.$watch(() => $ctrl.params.startDate, () => {
+            const endDate = new Date($ctrl.params.startDate);
+
+            endDate.setMonth(endDate.getMonth() + 1);
+
+            $ctrl.params.endDate = endDate;
+
+            reloadUsers($ctrl.params);
+        });
+    }
+
+    adjustHeight(rows) {
+        const height = Math.min(rows, 20) * 30 + 75;
+
+        // Remove header height.
+        this.gridApi.grid.element.css('height', height + 'px');
+
+        this.gridApi.core.handleWindowResize();
+    }
+
+    _enableColumns(_categories, visible) {
+        _.forEach(_categories, (cat) => {
+            cat.visible = visible;
+
+            _.forEach(this.gridOptions.columnDefs, (col) => {
+                if (col.categoryDisplayName === cat.name)
+                    col.visible = visible;
+            });
+        });
+
+        // Workaround for this.gridApi.grid.refresh() didn't return promise.
+        this.gridApi.grid.processColumnsProcessors(this.gridApi.grid.columns)
+            .then((renderableColumns) => this.gridApi.grid.setVisibleColumns(renderableColumns))
+            .then(() => this.gridApi.grid.redrawInPlace())
+            .then(() => this.gridApi.grid.refreshCanvas(true))
+            .then(() => {
+                if (visible) {
+                    const categoryDisplayName = _.last(_categories).name;
+
+                    const col = _.findLast(this.gridOptions.columnDefs, {categoryDisplayName});
+
+                    this.gridApi.grid.scrollTo(null, col);
+                }
+            });
+    }
+
+    _selectableColumns() {
+        return _.filter(this.gridOptions.categories, (cat) => cat.selectable);
+    }
+
+    toggleColumns(category, visible) {
+        this._enableColumns([category], visible);
+    }
+
+    selectAllColumns() {
+        this._enableColumns(this._selectableColumns(), true);
+    }
+
+    clearAllColumns() {
+        this._enableColumns(this._selectableColumns(), false);
+    }
+
+    exportCsv() {
+        this.gridApi.exporter.csvExport('all', 'visible');
+    }
+}
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
new file mode 100644
index 0000000000000..efed9c0edee16
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
@@ -0,0 +1,54 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins.jade
+include /app/components/form-field-datepicker/form-field-datepicker.jade
+
+mixin grid-settings()
+    i.fa.fa-bars(data-animation='am-flip-x' bs-dropdown='' aria-haspopup='true' aria-expanded='expanded' data-auto-close='1' data-trigger='click')
+    ul.select.dropdown-menu(role='menu')
+        li(ng-repeat='item in $ctrl.gridOptions.categories|filter:{selectable:true}')
+            a(ng-click='$ctrl.toggleColumns(item, !item.visible)')
+                i.fa.fa-check-square-o.pull-left(ng-if='item.visible')
+                i.fa.fa-square-o.pull-left(ng-if='!item.visible')
+                span {{::item.name}}
+        li.divider
+        li
+            a(ng-click='$ctrl.selectAllColumns()') Select all
+        li
+            a(ng-click='$ctrl.clearAllColumns()') Clear all
+        li.divider
+        li
+            a(ng-click='$hide()') Close
+
+.panel.panel-default
+    .panel-heading.ui-grid-settings
+        +grid-settings
+        label Total users: 
+            strong {{ $ctrl.gridOptions.data.length }}&nbsp;&nbsp;&nbsp;
+        label Showing users: 
+            strong {{ $ctrl.gridApi.grid.getVisibleRows().length }}
+            sub(ng-show='users.length === $ctrl.gridApi.grid.getVisibleRows().length') all
+        div.ui-grid-settings-dateperiod
+            form(ng-form=form novalidate)
+                -var form = 'admin'
+
+                +ignite-form-field-datepicker('Period:', '$ctrl.params.startDate', '"period"')
+                
+                button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
+
+    .panel-collapse
+        .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-exporter ui-grid-pinning)
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
new file mode 100644
index 0000000000000..7e44d94671452
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
@@ -0,0 +1,27 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+.ui-grid-header.ui-grid-header--subcategories(role='rowgroup')
+    .ui-grid-top-panel
+        .ui-grid-header-viewport
+            .ui-grid-header-canvas
+                .ui-grid-header-cell-wrapper(ng-style='colContainer.headerCellWrapperStyle()')
+                    .ui-grid-header-cell-row(role='row')
+                        .ui-grid-header-span.ui-grid-header-cell.ui-grid-clearfix(ng-repeat='cat in grid.options.categories')
+                            div(ng-show='(colContainer.renderedColumns|uiGridSubcategories:cat.name).length > 1')
+                                .ui-grid-cell-contents {{ cat.name }}
+                            .ui-grid-header-cell-row
+                                .ui-grid-header-cell.ui-grid-clearfix(ng-repeat='col in (colContainer.renderedColumns|uiGridSubcategories:cat.name) track by col.uid' ui-grid-header-cell='' col='col' render-index='$index')
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
new file mode 100644
index 0000000000000..c39050469db9c
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.ui-grid-header--subcategories {
+    .ui-grid-row:nth-child(even) .ui-grid-cell.cell-total {
+        background-color: rgba(102,175,233,.6);
+    }
+
+    .ui-grid-row:nth-child(odd) .ui-grid-cell.cell-total {
+        background-color: rgba(102,175,233,.3);
+    }
+
+    .ui-grid-header-cell-row {
+        height: 30px;
+    }
+
+    .ui-grid-header-cell [role="columnheader"] {
+        display: flex;
+        
+        flex-wrap: wrap;
+        align-items: center;
+        justify-content: center;
+
+        height: 100%;
+
+        & > div {
+            flex: 1 100%;
+            height: auto;
+        }
+
+        & > div[ui-grid-filter] {
+            flex: auto;
+        }
+    }
+
+    .ui-grid-header-span {
+        position: relative;
+        border-right: 0;
+
+        .ng-hide + .ui-grid-header-cell-row .ui-grid-header-cell {
+            height: 58px;
+        }
+
+        .ng-hide + .ui-grid-header-cell-row .ui-grid-cell-contents {
+            padding: 5px 5px;
+        }
+
+        .ui-grid-column-resizer.right {
+            top: -100px;
+        }
+        .ng-hide + .ui-grid-header-cell-row .ui-grid-column-resizer.right {
+            bottom: -100px;
+        }
+
+        &.ui-grid-header-cell .ui-grid-header-cell .ui-grid-column-resizer.right {
+            border-right-width: 0;
+        }
+        &.ui-grid-header-cell .ui-grid-header-cell:last-child .ui-grid-column-resizer.right {
+            border-right-width: 1px;
+        }
+
+        & > div > .ui-grid-cell-contents {
+            border-bottom: 1px solid #d4d4d4;
+        }
+    }
+
+    input {
+        line-height: 21px;
+    }
+}
diff --git a/modules/web-console/frontend/app/directives/ui-grid-settings/ui-grid-settings.jade b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.jade
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-grid-settings/ui-grid-settings.jade
rename to modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.jade
diff --git a/modules/web-console/frontend/app/directives/ui-grid-settings/ui-grid-settings.scss b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
similarity index 68%
rename from modules/web-console/frontend/app/directives/ui-grid-settings/ui-grid-settings.scss
rename to modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
index 6517a603e137a..3016488f6bb6e 100644
--- a/modules/web-console/frontend/app/directives/ui-grid-settings/ui-grid-settings.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
@@ -35,4 +35,36 @@
         padding-right: 8px;
         cursor: pointer;
     }
+
+    &-dateperiod {
+        float: right;
+
+        .ignite-form-field {
+            width: 160px;
+            margin-right: 10px;
+
+            &__label {
+            }
+
+            &__control {
+            }
+
+            &:nth-child(1) {
+                float: left;
+
+                .ignite-form-field__label {
+                    width: 40%;
+                }
+
+                .ignite-form-field__control {
+                    width: 60%;
+                }
+            }
+        }
+
+        .btn {
+            line-height: 20px;
+            margin-right: 0;
+        }
+    }
 }
diff --git a/modules/web-console/frontend/app/core/activities/Activities.data.js b/modules/web-console/frontend/app/core/activities/Activities.data.js
new file mode 100644
index 0000000000000..8a67a97a5dd43
--- /dev/null
+++ b/modules/web-console/frontend/app/core/activities/Activities.data.js
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class ActivitiesData {
+    static $inject = ['$http', '$state'];
+
+    constructor($http, $state) {
+        this.$http = $http;
+        this.$state = $state;
+    }
+
+    post(options = {}) {
+        let { group, action } = options;
+
+        action = action || this.$state.$current.url.source;
+        group = group || action.match(/^\/([^/]+)/)[1];
+
+        return this.$http.post('/api/v1/activities/page', { group, action });
+    }
+
+    listByUser(userId, params) {
+        return this.$http.get(`/api/v1/activities/user/${userId}`, { params })
+            .then(({ data }) => data);
+    }
+}
diff --git a/modules/web-console/frontend/app/core/admin/Admin.data.js b/modules/web-console/frontend/app/core/admin/Admin.data.js
new file mode 100644
index 0000000000000..66d82f0388518
--- /dev/null
+++ b/modules/web-console/frontend/app/core/admin/Admin.data.js
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class IgniteAdminData {
+    static $inject = ['$http', 'IgniteMessages', 'IgniteCountries'];
+
+    constructor($http, Messages, Countries) {
+        this.$http = $http;
+        this.Messages = Messages;
+        this.Countries = Countries;
+    }
+
+    becomeUser(viewedUserId) {
+        return this.$http.get('/api/v1/admin/become', {
+            params: {viewedUserId}
+        })
+        .catch(this.Messages.showError);
+    }
+
+    removeUser(user) {
+        return this.$http.post('/api/v1/admin/remove', {
+            userId: user._id
+        })
+        .then(() => {
+            this.Messages.showInfo(`User has been removed: "${user.userName}"`);
+        })
+        .catch(({data, status}) => {
+            if (status === 503)
+                this.Messages.showInfo(data);
+            else
+                this.Messages.showError('Failed to remove user: ', data);
+        });
+    }
+
+    toggleAdmin(user) {
+        return this.$http.post('/api/v1/admin/save', {
+            userId: user._id,
+            adminFlag: !user.admin
+        })
+        .then(() => {
+            this.Messages.showInfo(`Admin right was successfully toggled for user: "${user.userName}"`);
+        })
+        .catch((res) => {
+            this.Messages.showError('Failed to toggle admin right for user: ', res);
+        });
+    }
+
+    prepareUsers(user) {
+        const { Countries } = this;
+
+        user.userName = user.firstName + ' ' + user.lastName;
+        user.countryCode = Countries.getByName(user.country).code;
+
+        return user;
+    }
+
+    loadUsers(params) {
+        return this.$http.post('/api/v1/admin/list', params)
+            .then(({ data }) => data)
+            .then((users) => _.map(users, this.prepareUsers.bind(this)))
+            .catch(this.Messages.showError);
+    }
+}
diff --git a/modules/web-console/frontend/app/core/index.js b/modules/web-console/frontend/app/core/index.js
new file mode 100644
index 0000000000000..7f72ee32f811f
--- /dev/null
+++ b/modules/web-console/frontend/app/core/index.js
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import IgniteAdminData from './admin/Admin.data';
+import IgniteActivitiesData from './activities/Activities.data';
+
+angular.module('ignite-console.core', [])
+    .service('IgniteAdminData', IgniteAdminData)
+    .service('IgniteActivitiesData', IgniteActivitiesData);
diff --git a/modules/web-console/frontend/app/data/i18n.js b/modules/web-console/frontend/app/data/i18n.js
new file mode 100644
index 0000000000000..bc8c700d8bfa6
--- /dev/null
+++ b/modules/web-console/frontend/app/data/i18n.js
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default {
+    '/agent/start': 'Agent start',
+    '/agent/download': 'Agent download',
+    '/configuration/clusters': 'Configure clusters',
+    '/configuration/caches': 'Configure caches',
+    '/configuration/domains': 'Configure domain model',
+    '/configuration/igfs': 'Configure IGFS',
+    '/configuration/summary': 'Configurations summary',
+    '/demo/resume': 'Demo resume',
+    '/demo/reset': 'Demo reset',
+    '/queries/execute': 'Query execute',
+    '/queries/explain': 'Query explain',
+    '/queries/scan': 'Scan',
+    '/queries/add/query': 'Add query',
+    '/queries/add/scan': 'Add scan',
+    '/queries/demo': 'SQL demo',
+    '/queries/notebook/': 'Query notebook',
+    '/settings/profile': 'User profile',
+    '/settings/admin': 'Admin panel',
+    '/logout': 'Logout'
+};
diff --git a/modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js b/modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js
new file mode 100644
index 0000000000000..f36ae6e057023
--- /dev/null
+++ b/modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default [() => {
+    return (arr, category) => {
+        return _.filter(arr, (item) => {
+            return item.colDef.categoryDisplayName === category;
+        });
+    };
+}];
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 0d0b171df5595..4e972ef88ab6f 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -186,8 +186,8 @@ class Paragraph {
 }
 
 // Controller for SQL notebook screen.
-export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion',
-    function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version) {
+export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData',
+    function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData) {
         let stopTopology = null;
 
         const _tryStopRefresh = function(paragraph) {
@@ -965,6 +965,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
         $scope.addQuery = function() {
             const sz = $scope.notebook.paragraphs.length;
 
+            ActivitiesData.post({ action: '/queries/add/query' });
+
             const paragraph = new Paragraph($animate, $timeout, {
                 name: 'Query' + (sz === 0 ? '' : sz),
                 query: '',
@@ -991,6 +993,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
         $scope.addScan = function() {
             const sz = $scope.notebook.paragraphs.length;
 
+            ActivitiesData.post({ action: '/queries/add/scan' });
+
             const paragraph = new Paragraph($animate, $timeout, {
                 name: 'Scan' + (sz === 0 ? '' : sz),
                 query: '',
@@ -1379,6 +1383,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
                             const qry = args.maxPages ? addLimit(args.query, args.pageSize * args.maxPages) : paragraph.query;
 
+                            ActivitiesData.post({ action: '/queries/execute' });
+
                             return agentMonitor.query(nid, args.cacheName, qry, nonCollocatedJoins, local, args.pageSize);
                         })
                         .then((res) => {
@@ -1430,6 +1436,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                         pageSize: paragraph.pageSize
                     };
 
+                    ActivitiesData.post({ action: '/queries/explain' });
+
                     return agentMonitor.query(nid, args.cacheName, args.query, false, false, args.pageSize);
                 })
                 .then(_processQueryResult.bind(this, paragraph, true))
@@ -1466,6 +1474,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                                 localNid: local ? nid : null
                             };
 
+                            ActivitiesData.post({ action: '/queries/scan' });
+
                             return agentMonitor.query(nid, args.cacheName, query, false, local, args.pageSize);
                         })
                         .then((res) => _processQueryResult(paragraph, true, res))
diff --git a/modules/web-console/frontend/app/modules/sql/sql.module.js b/modules/web-console/frontend/app/modules/sql/sql.module.js
index a1ffde9646f99..5875961f0a896 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.module.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.module.js
@@ -30,7 +30,7 @@ angular.module('ignite-console.sql', [
             // set up the states
             $stateProvider
                 .state('base.sql', {
-                    url: '/sql',
+                    url: '/queries',
                     abstract: true,
                     template: '<ui-view></ui-view>'
                 })
diff --git a/modules/web-console/frontend/app/modules/states/admin.state.js b/modules/web-console/frontend/app/modules/states/admin.state.js
index c3151e1f442a8..35c6fbbfeeab0 100644
--- a/modules/web-console/frontend/app/modules/states/admin.state.js
+++ b/modules/web-console/frontend/app/modules/states/admin.state.js
@@ -29,7 +29,7 @@ angular
         templateUrl: '/settings/admin.html',
         onEnter: AclRoute.checkAccess('admin_page'),
         metaTags: {
-            title: 'List of registered users'
+            title: 'Admin panel'
         }
     });
 }]);
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
index cfc6df9c89538..16d2faef04335 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
@@ -21,8 +21,8 @@ import saver from 'file-saver';
 const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
 
 export default [
-    '$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteLoading', '$filter', 'IgniteConfigurationResource', 'JavaTypes', 'IgniteVersion', 'IgniteConfigurationGenerator', 'SpringTransformer', 'JavaTransformer', 'IgniteDockerGenerator', 'IgniteMavenGenerator', 'IgnitePropertiesGenerator', 'IgniteReadmeGenerator', 'IgniteFormUtils', 'IgniteSummaryZipper',
-    function($root, $scope, $http, LegacyUtils, Messages, Loading, $filter, Resource, JavaTypes, Version, generator, spring, java, docker, pom, propsGenerator, readme, FormUtils, SummaryZipper) {
+    '$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteLoading', '$filter', 'IgniteConfigurationResource', 'JavaTypes', 'IgniteVersion', 'IgniteConfigurationGenerator', 'SpringTransformer', 'JavaTransformer', 'IgniteDockerGenerator', 'IgniteMavenGenerator', 'IgnitePropertiesGenerator', 'IgniteReadmeGenerator', 'IgniteFormUtils', 'IgniteSummaryZipper', 'IgniteActivitiesData',
+    function($root, $scope, $http, LegacyUtils, Messages, Loading, $filter, Resource, JavaTypes, Version, generator, spring, java, docker, pom, propsGenerator, readme, FormUtils, SummaryZipper, ActivitiesData) {
         const ctrl = this;
 
         $scope.ui = {
@@ -304,6 +304,8 @@ export default [
 
             $scope.isPrepareDownloading = true;
 
+            ActivitiesData.post({ action: '/configuration/download' });
+
             return new SummaryZipper({ cluster, data: ctrl.data || {}, IgniteDemoMode: $root.IgniteDemoMode })
                 .then((data) => {
                     saver.saveAs(data, escapeFileName(cluster.name) + '-project.zip');
diff --git a/modules/web-console/frontend/app/modules/user/AclRoute.provider.js b/modules/web-console/frontend/app/modules/user/AclRoute.provider.js
index 40abea5ad872e..4225bc4ed3721 100644
--- a/modules/web-console/frontend/app/modules/user/AclRoute.provider.js
+++ b/modules/web-console/frontend/app/modules/user/AclRoute.provider.js
@@ -17,31 +17,36 @@
 
 export default [() => {
     class AclRoute {
-        static checkAccess = (permissions, failState) => {
+        static checkAccess(permissions, failState) {
             failState = failState || '403';
 
-            return ['$state', 'AclService', 'User', ($state, AclService, User) => {
-                User.read()
-                    .then(() => {
-                        if (AclService.can(permissions))
-                            return;
+            return ['$q', '$state', 'AclService', 'User', 'IgniteActivitiesData', function($q, $state, AclService, User, Activities) {
+                const action = this.name ? $state.href(this.name) : null;
 
-                        return $state.go(failState);
-                    })
+                return User.read()
                     .catch(() => {
                         User.clean();
 
                         if ($state.current.name !== 'signin')
                             $state.go('signin');
+
+                        return $q.reject('Failed to detect user');
+                    })
+                    .then(() => {
+                        if (AclService.can(permissions))
+                            return Activities.post({ action });
+
+                        $state.go(failState);
+
+                        return $q.reject('User are not authorized');
                     });
             }];
         }
-    }
 
-    return {
-        checkAccess: AclRoute.checkAccess,
-        $get: () => {
+        static $get() {
             return AclRoute;
         }
-    };
+    }
+
+    return AclRoute;
 }];
diff --git a/modules/web-console/frontend/app/modules/user/Auth.service.js b/modules/web-console/frontend/app/modules/user/Auth.service.js
index e0f905da8bf8b..95ff4c3e6dc4c 100644
--- a/modules/web-console/frontend/app/modules/user/Auth.service.js
+++ b/modules/web-console/frontend/app/modules/user/Auth.service.js
@@ -21,7 +21,7 @@ export default ['Auth', ['$http', '$rootScope', '$state', '$window', 'IgniteErro
             forgotPassword(userInfo) {
                 $http.post('/api/v1/password/forgot', userInfo)
                     .then(() => $state.go('password.send'))
-                    .cacth(({data}) => ErrorPopover.show('forgot_email', Messages.errorMessage(null, data)));
+                    .catch(({data}) => ErrorPopover.show('forgot_email', Messages.errorMessage(null, data)));
             },
             auth(action, userInfo) {
                 $http.post('/api/v1/' + action, userInfo)
diff --git a/modules/web-console/frontend/app/modules/user/permissions.js b/modules/web-console/frontend/app/modules/user/permissions.js
index e13509ceaf719..b6f7c3a30cf42 100644
--- a/modules/web-console/frontend/app/modules/user/permissions.js
+++ b/modules/web-console/frontend/app/modules/user/permissions.js
@@ -16,7 +16,7 @@
  */
 
 const guest = ['login'];
-const becomed = ['profile', 'configuration', 'query'];
+const becomed = ['profile', 'configuration', 'query', 'demo'];
 const user = becomed.concat(['logout']);
 const admin = user.concat(['admin_page']);
 
diff --git a/modules/web-console/frontend/app/modules/user/user.module.js b/modules/web-console/frontend/app/modules/user/user.module.js
index 11798d0746d62..b86a62e492066 100644
--- a/modules/web-console/frontend/app/modules/user/user.module.js
+++ b/modules/web-console/frontend/app/modules/user/user.module.js
@@ -22,10 +22,10 @@ import Auth from './Auth.service';
 import User from './User.service';
 import AclRouteProvider from './AclRoute.provider';
 
-angular
-.module('ignite-console.user', [
+angular.module('ignite-console.user', [
     'mm.acl',
-    'ignite-console.config'
+    'ignite-console.config',
+    'ignite-console.core'
 ])
 .factory('sessionRecoverer', ['$injector', '$q', ($injector, $q) => {
     return {
diff --git a/modules/web-console/frontend/app/vendor.js b/modules/web-console/frontend/app/vendor.js
index a9e88442cf0a3..3bbb3224ce3b2 100644
--- a/modules/web-console/frontend/app/vendor.js
+++ b/modules/web-console/frontend/app/vendor.js
@@ -25,6 +25,7 @@ import 'angular-strap/dist/angular-strap.tpl';
 import 'angular-socket-io';
 import 'angular-retina';
 import 'angular-ui-router';
+import 'angular-translate';
 import 'ui-router-metatags/dist/ui-router-metatags';
 import 'angular-smart-table';
 import 'angular-ui-grid/ui-grid';
diff --git a/modules/web-console/frontend/controllers/admin-controller.js b/modules/web-console/frontend/controllers/admin-controller.js
deleted file mode 100644
index cf7fd71adeec0..0000000000000
--- a/modules/web-console/frontend/controllers/admin-controller.js
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const ICON_SORT = '<span ui-grid-one-bind-id-grid="col.uid + \'-sortdir-text\'" ui-grid-visible="col.sort.direction" aria-label="Sort Descending"><i ng-class="{ \'ui-grid-icon-up-dir\': col.sort.direction == asc, \'ui-grid-icon-down-dir\': col.sort.direction == desc, \'ui-grid-icon-blank\': !col.sort.direction }" title="" aria-hidden="true"></i></span>';
-
-const CLUSTER_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-sitemap'></i>${ICON_SORT}</div>`;
-const MODEL_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-object-group'></i>${ICON_SORT}</div>`;
-const CACHE_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-database'></i>${ICON_SORT}</div>`;
-const IGFS_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-folder-o'></i>${ICON_SORT}</div>`;
-
-const ACTIONS_TEMPLATE = `
-<div class='text-center ui-grid-cell-actions'>
-    <a class='btn btn-default dropdown-toggle' bs-dropdown='' ng-show='row.entity._id != $root.user._id' data-placement='bottom-right' data-container='.panel'>
-        <i class='fa fa-gear'></i>&nbsp;
-        <span class='caret'></span>
-    </a>
-    <ul class='dropdown-menu' role='menu'>
-        <li>
-            <a ng-click='grid.api.becomeUser(row.entity)'>Become this user</a>
-        </li>
-        <li>
-            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='row.entity.admin && row.entity._id !== $root.user._id'>Revoke admin</a>
-            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='!row.entity.admin && row.entity._id !== $root.user._id'>Grant admin</a>
-        </li>
-        <li>
-            <a ng-click='grid.api.removeUser(row.entity)'>Remove user</a>
-        </li>
-</div>`;
-
-const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
-
-// Controller for Admin screen.
-export default ['adminController', [
-    '$rootScope', '$scope', '$http', '$q', '$state', '$filter', 'uiGridConstants', 'IgniteMessages', 'IgniteConfirm', 'User', 'IgniteNotebookData', 'IgniteCountries',
-    ($rootScope, $scope, $http, $q, $state, $filter, uiGridConstants, Messages, Confirm, User, Notebook, Countries) => {
-        $scope.users = null;
-
-        const companySelectOptions = [];
-        const countrySelectOptions = [];
-
-        const COLUMNS_DEFS = [
-            {displayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'test', minWidth: 80, width: 80, enableFiltering: false, enableSorting: false},
-            {displayName: 'User', field: 'userName', minWidth: 65, enableFiltering: true, filter: { placeholder: 'Filter by name...' }},
-            {displayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
-            {displayName: 'Company', field: 'company', minWidth: 160, filter: {
-                selectOptions: companySelectOptions, type: uiGridConstants.filter.SELECT, condition: uiGridConstants.filter.EXACT }
-            },
-            {displayName: 'Country', field: 'countryCode', minWidth: 80, filter: {
-                selectOptions: countrySelectOptions, type: uiGridConstants.filter.SELECT, condition: uiGridConstants.filter.EXACT }
-            },
-            {displayName: 'Last login', field: 'lastLogin', cellFilter: 'date:"medium"', minWidth: 175, width: 175, enableFiltering: false, sort: { direction: 'desc', priority: 0 }},
-            {displayName: 'Clusters count', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: '_clusters', type: 'number', headerTooltip: 'Clusters count', minWidth: 50, width: 50, enableFiltering: false},
-            {displayName: 'Models count', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: '_models', type: 'number', headerTooltip: 'Models count', minWidth: 50, width: 50, enableFiltering: false},
-            {displayName: 'Caches count', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: '_caches', type: 'number', headerTooltip: 'Caches count', minWidth: 50, width: 50, enableFiltering: false},
-            {displayName: 'IGFS count', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: '_igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 50, width: 50, enableFiltering: false}
-        ];
-
-        const ctrl = $scope.ctrl = {};
-
-        const becomeUser = function(user) {
-            $http.get('/api/v1/admin/become', { params: {viewedUserId: user._id}})
-                .then(() => User.load())
-                .then(() => $state.go('base.configuration.clusters'))
-                .then(() => Notebook.load())
-                .catch(Messages.showError);
-        };
-
-        const removeUser = (user) => {
-            Confirm.confirm(`Are you sure you want to remove user: "${user.userName}"?`)
-                .then(() => {
-                    $http.post('/api/v1/admin/remove', {userId: user._id})
-                        .then(() => {
-                            const i = _.findIndex($scope.users, (u) => u._id === user._id);
-
-                            if (i >= 0)
-                                $scope.users.splice(i, 1);
-
-                            Messages.showInfo(`User has been removed: "${user.userName}"`);
-                        })
-                        .catch(({data, status}) => {
-                            if (status === 503)
-                                Messages.showInfo(data);
-                            else
-                                Messages.showError('Failed to remove user: ', data);
-                        });
-                });
-        };
-
-        const toggleAdmin = (user) => {
-            if (user.adminChanging)
-                return;
-
-            user.adminChanging = true;
-
-            $http.post('/api/v1/admin/save', {userId: user._id, adminFlag: !user.admin})
-                .then(() => {
-                    user.admin = !user.admin;
-
-                    Messages.showInfo(`Admin right was successfully toggled for user: "${user.userName}"`);
-                })
-                .catch((res) => {
-                    Messages.showError('Failed to toggle admin right for user: ', res);
-                })
-                .finally(() => user.adminChanging = false);
-        };
-
-
-        ctrl.gridOptions = {
-            data: [],
-            columnVirtualizationThreshold: 30,
-            columnDefs: COLUMNS_DEFS,
-            categories: [
-                {name: 'Actions', visible: true, selectable: true},
-                {name: 'User', visible: true, selectable: true},
-                {name: 'Email', visible: true, selectable: true},
-                {name: 'Company', visible: true, selectable: true},
-                {name: 'Country', visible: true, selectable: true},
-                {name: 'Last login', visible: true, selectable: true},
-
-                {name: 'Clusters count', visible: true, selectable: true},
-                {name: 'Models count', visible: true, selectable: true},
-                {name: 'Caches count', visible: true, selectable: true},
-                {name: 'IGFS count', visible: true, selectable: true}
-            ],
-            enableFiltering: true,
-            enableRowSelection: false,
-            enableRowHeaderSelection: false,
-            enableColumnMenus: false,
-            multiSelect: false,
-            modifierKeysToMultiSelect: true,
-            noUnselect: true,
-            flatEntityAccess: true,
-            fastWatch: true,
-            onRegisterApi: (api) => {
-                ctrl.gridApi = api;
-
-                api.becomeUser = becomeUser;
-                api.removeUser = removeUser;
-                api.toggleAdmin = toggleAdmin;
-            }
-        };
-
-        /**
-         * Set grid height.
-         *
-         * @param {Number} rows Rows count.
-         * @private
-         */
-        const adjustHeight = (rows) => {
-            const height = Math.min(rows, 20) * 30 + 75;
-
-            // Remove header height.
-            ctrl.gridApi.grid.element.css('height', height + 'px');
-
-            ctrl.gridApi.core.handleWindowResize();
-        };
-
-        const usersToFilterOptions = (column) => {
-            return _.sortBy(
-                _.map(
-                    _.groupBy($scope.users, (usr) => {
-                        const fld = usr[column];
-
-                        return _.isNil(fld) ? fld : fld.toUpperCase();
-                    }),
-                    (arr, value) => ({label: `${_.head(arr)[column] || 'Not set'} (${arr.length})`, value})
-                ),
-                'value');
-        };
-
-        const _reloadUsers = () => {
-            $http.post('/api/v1/admin/list')
-                .then(({ data }) => {
-                    $scope.users = data;
-
-                    companySelectOptions.length = 0;
-                    countrySelectOptions.length = 0;
-
-                    _.forEach($scope.users, (user) => {
-                        user.userName = user.firstName + ' ' + user.lastName;
-                        user.countryCode = Countries.getByName(user.country).code;
-
-                        user._clusters = user.counters.clusters;
-                        user._models = user.counters.models;
-                        user._caches = user.counters.caches;
-                        user._igfs = user.counters.igfs;
-                    });
-
-                    companySelectOptions.push(...usersToFilterOptions('company'));
-                    countrySelectOptions.push(...usersToFilterOptions('countryCode'));
-
-                    $scope.ctrl.gridOptions.data = data;
-
-                    adjustHeight(data.length);
-                })
-                .catch(Messages.showError);
-        };
-
-        _reloadUsers();
-
-        const _enableColumns = (categories, visible) => {
-            _.forEach(categories, (cat) => {
-                cat.visible = visible;
-
-                _.forEach(ctrl.gridOptions.columnDefs, (col) => {
-                    if (col.displayName === cat.name)
-                        col.visible = visible;
-                });
-            });
-
-            ctrl.gridApi.grid.refresh();
-        };
-
-        const _selectableColumns = () => _.filter(ctrl.gridOptions.categories, (cat) => cat.selectable);
-
-        ctrl.toggleColumns = (category, visible) => _enableColumns([category], visible);
-        ctrl.selectAllColumns = () => _enableColumns(_selectableColumns(), true);
-        ctrl.clearAllColumns = () => _enableColumns(_selectableColumns(), false);
-    }
-]];
diff --git a/modules/web-console/frontend/controllers/domains-controller.js b/modules/web-console/frontend/controllers/domains-controller.js
index 303110e086696..bfffe92f62747 100644
--- a/modules/web-console/frontend/controllers/domains-controller.js
+++ b/modules/web-console/frontend/controllers/domains-controller.js
@@ -17,8 +17,8 @@
 
 // Controller for Domain model screen.
 export default ['domainsController', [
-    '$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteAgentMonitor', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes',
-    function($root, $scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Focus, Confirm, ConfirmBatch, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, IgniteAgentMonitor, LegacyTable, Resource, ErrorPopover, FormUtils, JavaTypes, SqlTypes) {
+    '$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteAgentMonitor', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes', 'IgniteActivitiesData',
+    function($root, $scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Focus, Confirm, ConfirmBatch, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, IgniteAgentMonitor, LegacyTable, Resource, ErrorPopover, FormUtils, JavaTypes, SqlTypes, ActivitiesData) {
         UnsavedChangesGuard.install($scope);
 
         const emptyDomain = {empty: true};
@@ -460,6 +460,14 @@ export default ['domainsController', [
                 $scope.importDomain.loadingOptions = LOADING_JDBC_DRIVERS;
 
                 IgniteAgentMonitor.startWatch({text: 'Back to Domain models', goal: 'import domain model from database'})
+                    .then(() => {
+                        ActivitiesData.post({
+                            group: 'configuration',
+                            action: 'configuration/import/model'
+                        });
+
+                        return true;
+                    })
                     .then(importDomainModal.$promise)
                     .then(importDomainModal.show)
                     .then(() => {
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index fd50d5b6e7552..ff52ba44cf9a2 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -44,6 +44,7 @@
     "angular-socket-io": "~0.7.0",
     "angular-strap": "~2.3.8",
     "angular-touch": "~1.5.9",
+    "angular-translate": "~2.13.1",
     "angular-tree-control": "~0.2.26",
     "angular-ui-grid": "~3.2.9",
     "angular-ui-router": "~0.3.1",
diff --git a/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss b/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss
index bfa6c6ce78aad..47555a7c95750 100644
--- a/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss
+++ b/modules/web-console/frontend/public/stylesheets/_font-awesome-custom.scss
@@ -69,3 +69,31 @@ $fa-font-path: '~font-awesome/fonts';
 
   cursor: default;
 }
+
+.icon-user {
+  @extend .fa;
+  @extend .fa-user-o;
+
+  cursor: default;
+}
+
+.icon-admin {
+  @extend .fa;
+  @extend .fa-user-secret;
+
+  cursor: default;
+}
+
+.icon-datepicker-left {
+  @extend .fa;
+  @extend .fa-chevron-left;
+
+  margin: 0;
+}
+
+.icon-datepicker-right {
+  @extend .fa;
+  @extend .fa-chevron-right;
+  
+  margin: 0;
+}
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index 4318fc2abc5eb..67cfed176d39e 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -2302,12 +2302,13 @@ html,body,.splash-screen {
         cursor: default;
 
         i {
+            margin-top: 2px;
             margin-right: 10px;
         }
 
         label {
             cursor: default;
-            line-height: 24px;
+            line-height: 28px;
         }
 
         sub {
@@ -2326,4 +2327,40 @@ html,body,.splash-screen {
     .ui-grid-filter-select {
         width: calc(100% - 10px);
     }
+
+    .ui-grid-cell-contents > i {
+        line-height: $line-height-base;
+    }
+
+    .ui-grid-row:nth-child(odd):hover .ui-grid-cell {
+        background: $ignite-row-hover;
+    }
+    
+    .ui-grid-row:nth-child(even):hover .ui-grid-cell {
+        background: $ignite-row-hover;
+    }
 }
+
+.datepicker.dropdown-menu {
+    width: 250px;
+    height: 270px;
+
+    button {
+        outline: none;
+        border: 0;
+    }
+
+    tbody {
+        height: 180px;
+    }
+
+    tbody button {
+        padding: 6px;
+    }
+
+    &.datepicker-mode-1, &.datepicker-mode-2 {
+        tbody button {
+            height: 65px;
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/public/stylesheets/variables.scss b/modules/web-console/frontend/public/stylesheets/variables.scss
index 8500eace44e41..e30bbddf9a6e4 100644
--- a/modules/web-console/frontend/public/stylesheets/variables.scss
+++ b/modules/web-console/frontend/public/stylesheets/variables.scss
@@ -26,3 +26,4 @@ $ignite-border-bottom-color: $brand-primary;
 $ignite-background-color: #fff;
 $ignite-header-color: #555;
 $ignite-invalid-color: $brand-primary;
+$ignite-row-hover: #c9dde1;
diff --git a/modules/web-console/frontend/views/settings/admin.jade b/modules/web-console/frontend/views/settings/admin.jade
index c9858269eeaf3..a09fda95a9eb1 100644
--- a/modules/web-console/frontend/views/settings/admin.jade
+++ b/modules/web-console/frontend/views/settings/admin.jade
@@ -14,38 +14,12 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin grid-settings()
-    i.fa.fa-bars(data-animation='am-flip-x' bs-dropdown='' aria-haspopup='true' aria-expanded='expanded' data-auto-close='1' data-trigger='click')
-    ul.select.dropdown-menu(role='menu')
-        li(ng-repeat='item in ctrl.gridOptions.categories|filter:{selectable:true}')
-            a(ng-click='ctrl.toggleColumns(item, !item.visible)')
-                i.fa.fa-check-square-o.pull-left(ng-if='item.visible')
-                i.fa.fa-square-o.pull-left(ng-if='!item.visible')
-                span {{::item.name}}
-        li.divider
-        li
-            a(ng-click='ctrl.selectAllColumns()') Select all
-        li
-            a(ng-click='ctrl.clearAllColumns()') Clear all
-        li.divider
-        li
-            a(ng-click='$hide()') Close
-
-.admin-page.row(ng-controller='adminController')
+.admin-page.row
     .docs-content.greedy
         .docs-header
-            h1 List of registered users
+            h1 Admin panel
             hr
         .docs-body
             .row
                 .col-xs-12
-                    .panel.panel-default
-                        .panel-heading.ui-grid-settings
-                            +grid-settings
-                            label Total users: 
-                                strong {{ users.length }}&nbsp;&nbsp;&nbsp;
-                            label Showing users: 
-                                strong {{ ctrl.gridApi.grid.getVisibleRows().length }}
-                                sub(ng-show='users.length === ctrl.gridApi.grid.getVisibleRows().length') all
-                        .panel-collapse
-                            .grid(ui-grid='ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-pinning)
+                    ignite-list-of-registered-users(data-options='ctrl.data')
diff --git a/modules/web-console/frontend/views/sql/sql.jade b/modules/web-console/frontend/views/sql/sql.jade
index 03015e8aad612..61d5b3037825a 100644
--- a/modules/web-console/frontend/views/sql/sql.jade
+++ b/modules/web-console/frontend/views/sql/sql.jade
@@ -15,7 +15,7 @@
     limitations under the License.
 
 include /app/helpers/jade/mixins.jade
-include /app/directives/ui-grid-settings/ui-grid-settings.jade
+include /app/components/ui-grid-settings/ui-grid-settings.jade
 
 mixin btn-toolbar(btn, click, tip, focusId)
     i.btn.btn-default.fa(class=btn ng-click=click bs-tooltip='' data-title=tip ignite-on-click-focus=focusId data-trigger='hover' data-placement='bottom')
@@ -195,7 +195,7 @@ mixin paragraph-scan
                 +table-result-body
             .footer.clearfix()
                 .pull-left
-                    | Showing results for scan of #[b{{ paragraph.queryArgs.cacheName | defaultName }}]
+                    | Showing results for scan of #[b {{ paragraph.queryArgs.cacheName | defaultName }}]
                     span(ng-if='paragraph.queryArgs.filter') &nbsp; with filter: #[b {{ paragraph.queryArgs.filter }}]
                     span(ng-if='paragraph.queryArgs.localNid') &nbsp; on node: #[b {{ paragraph.queryArgs.localNid | limitTo:8 }}]
 

From b571fd40f009709718bce0c969d47b62be06b8f6 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 8 Feb 2017 17:05:08 +0700
Subject: [PATCH 013/516] IGNITE-4472 Added user activities in Web Console.

(cherry picked from commit 26ee9c2)
---
 .../frontend/app/modules/{Demo => demo}/Demo.module.js            | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename modules/web-console/frontend/app/modules/{Demo => demo}/Demo.module.js (100%)

diff --git a/modules/web-console/frontend/app/modules/Demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js
similarity index 100%
rename from modules/web-console/frontend/app/modules/Demo/Demo.module.js
rename to modules/web-console/frontend/app/modules/demo/Demo.module.js

From db48f545a83bb72cded77d8269fe4f8820e8380f Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 10 Feb 2017 15:55:05 +0700
Subject: [PATCH 014/516] IGNITE-4678 Web Console: Implemented demo load as
 service.

(cherry picked from commit a600caf)
---
 modules/web-console/backend/app/agent.js      |  34 ++
 modules/web-console/backend/app/browser.js    |  26 +
 modules/web-console/frontend/package.json     |   2 +-
 .../ignite/console/demo/AgentClusterDemo.java | 475 +-----------------
 .../ignite/console/demo/AgentDemoUtils.java   |  79 +++
 .../demo/service/DemoCachesLoadService.java   | 456 +++++++++++++++++
 .../service/DemoRandomCacheLoadService.java   | 120 +++++
 .../service/DemoServiceClusterSingleton.java  |  41 ++
 .../demo/service/DemoServiceKeyAffinity.java  |  41 ++
 .../service/DemoServiceMultipleInstances.java |  41 ++
 .../service/DemoServiceNodeSingleton.java     |  41 ++
 11 files changed, 897 insertions(+), 459 deletions(-)
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentDemoUtils.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceClusterSingleton.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceKeyAffinity.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceMultipleInstances.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceNodeSingleton.java

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index 817028059f50f..4cae8ee124175 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -581,6 +581,40 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo,
 
             return this.executeRest(cmd);
         }
+
+        /**
+         * Collect service information.
+         * @param {Boolean} demo Is need run command on demo node.
+         * @param {String} nid Node ID.
+         * @returns {Promise}
+         */
+        services(demo, nid) {
+            const cmd = new Command(demo, 'exe')
+                .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
+                .addParam('p1', nid)
+                .addParam('p2', 'org.apache.ignite.internal.visor.service.VisorServiceTask')
+                .addParam('p3', 'java.lang.Void');
+
+            return this.executeRest(cmd);
+        }
+
+        /**
+         * Cancel service with specified name.
+         * @param {Boolean} demo Is need run command on demo node.
+         * @param {String} nid Node ID.
+         * @param {String} name Name of service to cancel.
+         * @returns {Promise}
+         */
+        serviceCancel(demo, nid, name) {
+            const cmd = new Command(demo, 'exe')
+                .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
+                .addParam('p1', nid)
+                .addParam('p2', 'org.apache.ignite.internal.visor.service.VisorCancelServiceTask')
+                .addParam('p3', 'java.lang.String')
+                .addParam('p4', name);
+
+            return this.executeRest(cmd);
+        }
     }
 
     /**
diff --git a/modules/web-console/backend/app/browser.js b/modules/web-console/backend/app/browser.js
index 499d84d8e990a..2b1285e4f79c1 100644
--- a/modules/web-console/backend/app/browser.js
+++ b/modules/web-console/backend/app/browser.js
@@ -455,6 +455,32 @@ module.exports.factory = (_, socketio, agentMgr, configure) => {
                         .catch((err) => cb(_errorToJson(err)));
                 });
 
+                // Collect service information from grid.
+                socket.on('service:collect', (nid, cb) => {
+                    agentMgr.findAgent(accountId())
+                        .then((agent) => agent.services(demo, nid))
+                        .then((data) => {
+                            if (data.finished)
+                                return cb(null, data.result);
+
+                            cb(_errorToJson(data.error));
+                        })
+                        .catch((err) => cb(_errorToJson(err)));
+                });
+
+                // Collect service information from grid.
+                socket.on('service:cancel', (nid, name, cb) => {
+                    agentMgr.findAgent(accountId())
+                        .then((agent) => agent.serviceCancel(demo, nid, name))
+                        .then((data) => {
+                            if (data.finished)
+                                return cb(null, data.result);
+
+                            cb(_errorToJson(data.error));
+                        })
+                        .catch((err) => cb(_errorToJson(err)));
+                });
+
                 const count = agentMgr.addAgentListener(user._id, socket);
 
                 socket.emit('agent:count', {count});
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index ff52ba44cf9a2..651f496fa78c7 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -46,7 +46,7 @@
     "angular-touch": "~1.5.9",
     "angular-translate": "~2.13.1",
     "angular-tree-control": "~0.2.26",
-    "angular-ui-grid": "~3.2.9",
+    "angular-ui-grid": "~4.0.0",
     "angular-ui-router": "~0.3.1",
     "bootstrap-sass": "~3.3.6",
     "brace": "~0.8.0",
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
index 489e762eb5362..252692eef57fa 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
@@ -17,37 +17,24 @@
 
 package org.apache.ignite.console.demo;
 
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.Random;
-import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteServices;
 import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.QueryEntity;
-import org.apache.ignite.cache.QueryIndex;
-import org.apache.ignite.cache.QueryIndexType;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.console.agent.AgentConfiguration;
-import org.apache.ignite.console.demo.model.Car;
-import org.apache.ignite.console.demo.model.Country;
-import org.apache.ignite.console.demo.model.Department;
-import org.apache.ignite.console.demo.model.Employee;
-import org.apache.ignite.console.demo.model.Parking;
+import org.apache.ignite.console.demo.service.DemoCachesLoadService;
+import org.apache.ignite.console.demo.service.DemoRandomCacheLoadService;
+import org.apache.ignite.console.demo.service.DemoServiceMultipleInstances;
+import org.apache.ignite.console.demo.service.DemoServiceClusterSingleton;
+import org.apache.ignite.console.demo.service.DemoServiceKeyAffinity;
+import org.apache.ignite.console.demo.service.DemoServiceNodeSingleton;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.logger.log4j.Log4JLogger;
@@ -55,7 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.spi.swapspace.file.FileSwapSpaceSpi;
-import org.apache.ignite.transactions.Transaction;
 import org.apache.log4j.Logger;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE;
@@ -66,8 +52,6 @@
 import static org.apache.ignite.events.EventType.EVTS_DISCOVERY;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_JETTY_ADDRS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_JETTY_PORT;
-import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
-import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
 /**
  * Demo for cluster features like SQL and Monitoring.
@@ -84,247 +68,6 @@ public class AgentClusterDemo {
     /** */
     private static final int NODE_CNT = 3;
 
-    /** */
-    private static final String COUNTRY_CACHE_NAME = "CountryCache";
-
-    /** */
-    private static final String DEPARTMENT_CACHE_NAME = "DepartmentCache";
-
-    /** */
-    private static final String EMPLOYEE_CACHE_NAME = "EmployeeCache";
-
-    /** */
-    private static final String PARKING_CACHE_NAME = "ParkingCache";
-
-    /** */
-    private static final String CAR_CACHE_NAME = "CarCache";
-
-    /** */
-    private static final Set<String> DEMO_CACHES = new HashSet<>(Arrays.asList(COUNTRY_CACHE_NAME,
-        DEPARTMENT_CACHE_NAME, EMPLOYEE_CACHE_NAME, PARKING_CACHE_NAME, CAR_CACHE_NAME));
-
-    /** */
-    private static final Random rnd = new Random();
-
-    /** Countries count. */
-    private static final int CNTR_CNT = 10;
-
-    /** Departments count */
-    private static final int DEP_CNT = 100;
-
-    /** Employees count. */
-    private static final int EMPL_CNT = 1000;
-
-    /** Countries count. */
-    private static final int CAR_CNT = 100;
-
-    /** Departments count */
-    private static final int PARK_CNT = 10;
-
-    /** Counter for threads in pool. */
-    private static final AtomicInteger THREAD_CNT = new AtomicInteger(0);
-
-    /**
-     * Create base cache configuration.
-     *
-     * @param name cache name.
-     * @return Cache configuration with basic properties set.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheConfiguration(String name) {
-        CacheConfiguration<K, V> ccfg = new CacheConfiguration<>(name);
-
-        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-        ccfg.setQueryDetailMetricsSize(10);
-        ccfg.setStartSize(100);
-        ccfg.setStatisticsEnabled(true);
-
-        return ccfg;
-    }
-
-    /**
-     * Configure cacheCountry.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheCountry() {
-        CacheConfiguration<K, V> ccfg = cacheConfiguration(COUNTRY_CACHE_NAME);
-
-        // Configure cacheCountry types.
-        Collection<QueryEntity> qryEntities = new ArrayList<>();
-
-        // COUNTRY.
-        QueryEntity type = new QueryEntity();
-
-        qryEntities.add(type);
-
-        type.setKeyType(Integer.class.getName());
-        type.setValueType(Country.class.getName());
-
-        // Query fields for COUNTRY.
-        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
-
-        qryFlds.put("id", "java.lang.Integer");
-        qryFlds.put("name", "java.lang.String");
-        qryFlds.put("population", "java.lang.Integer");
-
-        type.setFields(qryFlds);
-
-        ccfg.setQueryEntities(qryEntities);
-
-        return ccfg;
-    }
-
-    /**
-     * Configure cacheEmployee.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheDepartment() {
-        CacheConfiguration<K, V> ccfg = cacheConfiguration(DEPARTMENT_CACHE_NAME);
-
-        // Configure cacheDepartment types.
-        Collection<QueryEntity> qryEntities = new ArrayList<>();
-
-        // DEPARTMENT.
-        QueryEntity type = new QueryEntity();
-
-        qryEntities.add(type);
-
-        type.setKeyType(Integer.class.getName());
-        type.setValueType(Department.class.getName());
-
-        // Query fields for DEPARTMENT.
-        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
-
-        qryFlds.put("id", "java.lang.Integer");
-        qryFlds.put("countryId", "java.lang.Integer");
-        qryFlds.put("name", "java.lang.String");
-
-        type.setFields(qryFlds);
-
-        ccfg.setQueryEntities(qryEntities);
-
-        return ccfg;
-    }
-
-    /**
-     * Configure cacheEmployee.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheEmployee() {
-        CacheConfiguration<K, V> ccfg = cacheConfiguration(EMPLOYEE_CACHE_NAME);
-
-        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
-        ccfg.setBackups(1);
-
-        // Configure cacheEmployee types.
-        Collection<QueryEntity> qryEntities = new ArrayList<>();
-
-        // EMPLOYEE.
-        QueryEntity type = new QueryEntity();
-
-        qryEntities.add(type);
-
-        type.setKeyType(Integer.class.getName());
-        type.setValueType(Employee.class.getName());
-
-        // Query fields for EMPLOYEE.
-        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
-
-        qryFlds.put("id", "java.lang.Integer");
-        qryFlds.put("departmentId", "java.lang.Integer");
-        qryFlds.put("managerId", "java.lang.Integer");
-        qryFlds.put("firstName", "java.lang.String");
-        qryFlds.put("lastName", "java.lang.String");
-        qryFlds.put("email", "java.lang.String");
-        qryFlds.put("phoneNumber", "java.lang.String");
-        qryFlds.put("hireDate", "java.sql.Date");
-        qryFlds.put("job", "java.lang.String");
-        qryFlds.put("salary", "java.lang.Double");
-
-        type.setFields(qryFlds);
-
-        // Indexes for EMPLOYEE.
-        Collection<QueryIndex> indexes = new ArrayList<>();
-
-        QueryIndex idx = new QueryIndex();
-
-        idx.setName("EMP_NAMES");
-        idx.setIndexType(QueryIndexType.SORTED);
-        LinkedHashMap<String, Boolean> indFlds = new LinkedHashMap<>();
-
-        indFlds.put("firstName", Boolean.FALSE);
-        indFlds.put("lastName", Boolean.FALSE);
-
-        idx.setFields(indFlds);
-
-        indexes.add(idx);
-        indexes.add(new QueryIndex("salary", QueryIndexType.SORTED, false, "EMP_SALARY"));
-
-        type.setIndexes(indexes);
-
-        ccfg.setQueryEntities(qryEntities);
-
-        return ccfg;
-    }
-
-    /**
-     * Configure cacheEmployee.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheParking() {
-        CacheConfiguration<K, V> ccfg = cacheConfiguration(PARKING_CACHE_NAME);
-
-        // Configure cacheParking types.
-        Collection<QueryEntity> qryEntities = new ArrayList<>();
-
-        // PARKING.
-        QueryEntity type = new QueryEntity();
-
-        qryEntities.add(type);
-
-        type.setKeyType(Integer.class.getName());
-        type.setValueType(Parking.class.getName());
-
-        // Query fields for PARKING.
-        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
-
-        qryFlds.put("id", "java.lang.Integer");
-        qryFlds.put("name", "java.lang.String");
-        qryFlds.put("capacity", "java.lang.Integer");
-
-        type.setFields(qryFlds);
-
-        ccfg.setQueryEntities(qryEntities);
-
-        return ccfg;
-    }
-
-    /**
-     * Configure cacheEmployee.
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheCar() {
-        CacheConfiguration<K, V> ccfg = cacheConfiguration(CAR_CACHE_NAME);
-
-        // Configure cacheCar types.
-        Collection<QueryEntity> qryEntities = new ArrayList<>();
-
-        // CAR.
-        QueryEntity type = new QueryEntity();
-
-        qryEntities.add(type);
-
-        type.setKeyType(Integer.class.getName());
-        type.setValueType(Car.class.getName());
-
-        // Query fields for CAR.
-        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
-
-        qryFlds.put("id", "java.lang.Integer");
-        qryFlds.put("parkingId", "java.lang.Integer");
-        qryFlds.put("name", "java.lang.String");
-
-        type.setFields(qryFlds);
-
-        ccfg.setQueryEntities(qryEntities);
-
-        return ccfg;
-    }
-
     /**
      * Configure node.
      * @param gridIdx Grid name index.
@@ -363,120 +106,11 @@ private static  IgniteConfiguration igniteConfiguration(int gridIdx, boolean cli
         if (client)
             cfg.setClientMode(true);
 
-        cfg.setCacheConfiguration(cacheCountry(), cacheDepartment(), cacheEmployee(), cacheParking(), cacheCar());
-
         cfg.setSwapSpaceSpi(new FileSwapSpaceSpi());
 
         return cfg;
     }
 
-    /**
-     * @param val Value to round.
-     * @param places Numbers after point.
-     * @return Rounded value;
-     */
-    private static double round(double val, int places) {
-        if (places < 0)
-            throw new IllegalArgumentException();
-
-        long factor = (long)Math.pow(10, places);
-
-        val *= factor;
-
-        long tmp = Math.round(val);
-
-        return (double)tmp / factor;
-    }
-
-    /**
-     * @param ignite Ignite.
-     * @param range Time range in milliseconds.
-     */
-    private static void populateCacheEmployee(Ignite ignite, long range) {
-        if (log.isDebugEnabled())
-            log.debug("DEMO: Start employees population with data...");
-
-        IgniteCache<Integer, Country> cacheCountry = ignite.cache(COUNTRY_CACHE_NAME);
-
-        for (int i = 0, n = 1; i < CNTR_CNT; i++, n++)
-            cacheCountry.put(i, new Country(i, "Country #" + n, n * 10000000));
-
-        IgniteCache<Integer, Department> cacheDepartment = ignite.cache(DEPARTMENT_CACHE_NAME);
-
-        IgniteCache<Integer, Employee> cacheEmployee = ignite.cache(EMPLOYEE_CACHE_NAME);
-
-        for (int i = 0, n = 1; i < DEP_CNT; i++, n++) {
-            cacheDepartment.put(i, new Department(n, rnd.nextInt(CNTR_CNT), "Department #" + n));
-
-            double r = rnd.nextDouble();
-
-            cacheEmployee.put(i, new Employee(i, rnd.nextInt(DEP_CNT), null, "First name manager #" + n,
-                "Last name manager #" + n, "Email manager #" + n, "Phone number manager #" + n,
-                new java.sql.Date((long)(r * range)), "Job manager #" + n, 1000 + round(r * 4000, 2)));
-        }
-
-        for (int i = 0, n = 1; i < EMPL_CNT; i++, n++) {
-            Integer depId = rnd.nextInt(DEP_CNT);
-
-            double r = rnd.nextDouble();
-
-            cacheEmployee.put(i, new Employee(i, depId, depId, "First name employee #" + n,
-                "Last name employee #" + n, "Email employee #" + n, "Phone number employee #" + n,
-                new java.sql.Date((long)(r * range)), "Job employee #" + n, 500 + round(r * 2000, 2)));
-        }
-
-        if (log.isDebugEnabled())
-            log.debug("DEMO: Finished employees population.");
-    }
-
-    /**
-     * @param ignite Ignite.
-     */
-    private static void populateCacheCar(Ignite ignite) {
-        if (log.isDebugEnabled())
-            log.debug("DEMO: Start cars population...");
-
-        IgniteCache<Integer, Parking> cacheParking = ignite.cache(PARKING_CACHE_NAME);
-
-        for (int i = 0, n = 1; i < PARK_CNT; i++, n++)
-            cacheParking.put(i, new Parking(i, "Parking #" + n, n * 10));
-
-        IgniteCache<Integer, Car> cacheCar = ignite.cache(CAR_CACHE_NAME);
-
-        for (int i = 0, n = 1; i < CAR_CNT; i++, n++)
-            cacheCar.put(i, new Car(i, rnd.nextInt(PARK_CNT), "Car #" + n));
-
-        if (log.isDebugEnabled())
-            log.debug("DEMO: Finished cars population.");
-    }
-
-    /**
-     * Creates a thread pool that can schedule commands to run after a given delay, or to execute periodically.
-     *
-     * @param corePoolSize Number of threads to keep in the pool, even if they are idle.
-     * @param threadName Part of thread name that would be used by thread factory.
-     * @return Newly created scheduled thread pool.
-     */
-    private static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, final String threadName) {
-        ScheduledExecutorService srvc = Executors.newScheduledThreadPool(corePoolSize, new ThreadFactory() {
-            @Override public Thread newThread(Runnable r) {
-                Thread thread = new Thread(r, String.format("%s-%d", threadName, THREAD_CNT.getAndIncrement()));
-
-                thread.setDaemon(true);
-
-                return thread;
-            }
-        });
-
-        ScheduledThreadPoolExecutor executor = (ScheduledThreadPoolExecutor)srvc;
-
-        // Setting up shutdown policy.
-        executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
-        executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
-
-        return srvc;
-    }
-
     /**
      * Starts read and write from cache in background.
      *
@@ -484,91 +118,8 @@ private static ScheduledExecutorService newScheduledThreadPool(int corePoolSize,
      * @param cnt - maximum count read/write key
      */
     private static void startLoad(final Ignite ignite, final int cnt) {
-        final long diff = new java.util.Date().getTime();
-
-        populateCacheEmployee(ignite, diff);
-        populateCacheCar(ignite);
-
-        ScheduledExecutorService cachePool = newScheduledThreadPool(2, "demo-sql-load-cache-tasks");
-
-        cachePool.scheduleWithFixedDelay(new Runnable() {
-            @Override public void run() {
-                try {
-                    for (String cacheName : ignite.cacheNames()) {
-                        if (!DEMO_CACHES.contains(cacheName)) {
-                            IgniteCache<Integer, String> otherCache = ignite.cache(cacheName);
-
-                            if (otherCache != null) {
-                                for (int i = 0, n = 1; i < cnt; i++, n++) {
-                                    Integer key = rnd.nextInt(1000);
-
-                                    String val = otherCache.get(key);
-
-                                    if (val == null)
-                                        otherCache.put(key, "other-" + key);
-                                    else if (rnd.nextInt(100) < 30)
-                                        otherCache.remove(key);
-                                }
-                            }
-                        }
-                    }
-
-                    IgniteCache<Integer, Employee> cacheEmployee = ignite.cache(EMPLOYEE_CACHE_NAME);
-
-                    if (cacheEmployee != null)
-                        try(Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
-                            for (int i = 0, n = 1; i < cnt; i++, n++) {
-                                Integer id = rnd.nextInt(EMPL_CNT);
-
-                                Integer depId = rnd.nextInt(DEP_CNT);
-
-                                double r = rnd.nextDouble();
-
-                                cacheEmployee.put(id, new Employee(id, depId, depId, "First name employee #" + n,
-                                    "Last name employee #" + n, "Email employee #" + n, "Phone number employee #" + n,
-                                    new java.sql.Date((long)(r * diff)), "Job employee #" + n, 500 + round(r * 2000, 2)));
-
-                                if (rnd.nextBoolean())
-                                    cacheEmployee.remove(rnd.nextInt(EMPL_CNT));
-
-                                cacheEmployee.get(rnd.nextInt(EMPL_CNT));
-                            }
-
-                            if (rnd.nextInt(100) > 20)
-                                tx.commit();
-                        }
-                }
-                catch (Throwable e) {
-                    if (!e.getMessage().contains("cache is stopped"))
-                        ignite.log().error("Cache write task execution error", e);
-                }
-            }
-        }, 10, 3, TimeUnit.SECONDS);
-
-        cachePool.scheduleWithFixedDelay(new Runnable() {
-            @Override public void run() {
-                try {
-                    IgniteCache<Integer, Car> cache = ignite.cache(CAR_CACHE_NAME);
-
-                    if (cache != null)
-                        for (int i = 0; i < cnt; i++) {
-                            Integer carId = rnd.nextInt(CAR_CNT);
-
-                            cache.put(carId, new Car(carId, rnd.nextInt(PARK_CNT), "Car #" + (i + 1)));
-
-                            if (rnd.nextBoolean())
-                                cache.remove(rnd.nextInt(CAR_CNT));
-                        }
-                }
-                catch (IllegalStateException ignored) {
-                    // No-op.
-                }
-                catch (Throwable e) {
-                    if (!e.getMessage().contains("cache is stopped"))
-                        ignite.log().error("Cache write task execution error", e);
-                }
-            }
-        }, 10, 3, TimeUnit.SECONDS);
+        ignite.services().deployClusterSingleton("Demo caches load service", new DemoCachesLoadService(cnt));
+        ignite.services().deployNodeSingleton("RandomCache load service", new DemoRandomCacheLoadService(cnt));
     }
 
     /**
@@ -609,6 +160,14 @@ public static boolean testDrive(AgentConfiguration acfg) {
                     }
                 }, 10, 10, TimeUnit.SECONDS);
 
+                IgniteServices services = ignite.services();
+
+                services.deployMultiple("Demo service: Multiple instances", new DemoServiceMultipleInstances(), 7, 3);
+                services.deployNodeSingleton("Demo service: Node singleton", new DemoServiceNodeSingleton());
+                services.deployClusterSingleton("Demo service: Cluster singleton", new DemoServiceClusterSingleton());
+                services.deployKeyAffinitySingleton("Demo service: Key affinity singleton",
+                    new DemoServiceKeyAffinity(), DemoCachesLoadService.CAR_CACHE_NAME, "id");
+
                 if (log.isDebugEnabled())
                     log.debug("DEMO: Started embedded nodes with indexed enabled caches...");
 
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentDemoUtils.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentDemoUtils.java
new file mode 100644
index 0000000000000..fb34de7e8b41f
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentDemoUtils.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Utilites for Agent demo mode.
+ */
+public class AgentDemoUtils {
+    /** Counter for threads in pool. */
+    private static final AtomicInteger THREAD_CNT = new AtomicInteger(0);
+
+    /**
+     * Creates a thread pool that can schedule commands to run after a given delay, or to execute periodically.
+     *
+     * @param corePoolSize Number of threads to keep in the pool, even if they are idle.
+     * @param threadName Part of thread name that would be used by thread factory.
+     * @return Newly created scheduled thread pool.
+     */
+    public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, final String threadName) {
+        ScheduledExecutorService srvc = Executors.newScheduledThreadPool(corePoolSize, new ThreadFactory() {
+            @Override public Thread newThread(Runnable r) {
+                Thread thread = new Thread(r, String.format("%s-%d", threadName, THREAD_CNT.getAndIncrement()));
+
+                thread.setDaemon(true);
+
+                return thread;
+            }
+        });
+
+        ScheduledThreadPoolExecutor executor = (ScheduledThreadPoolExecutor)srvc;
+
+        // Setting up shutdown policy.
+        executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
+        executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
+
+        return srvc;
+    }
+
+    /**
+     * Round value.
+     *
+     * @param val Value to round.
+     * @param places Numbers after point.
+     * @return Rounded value;
+     */
+    public static double round(double val, int places) {
+        if (places < 0)
+            throw new IllegalArgumentException();
+
+        long factor = (long)Math.pow(10, places);
+
+        val *= factor;
+
+        long tmp = Math.round(val);
+
+        return (double)tmp / factor;
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
new file mode 100644
index 0000000000000..911764604502f
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
@@ -0,0 +1,456 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.cache.QueryIndexType;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.console.demo.AgentDemoUtils;
+import org.apache.ignite.console.demo.model.Car;
+import org.apache.ignite.console.demo.model.Country;
+import org.apache.ignite.console.demo.model.Department;
+import org.apache.ignite.console.demo.model.Employee;
+import org.apache.ignite.console.demo.model.Parking;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.transactions.Transaction;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Demo service. Create and populate caches. Run demo load on caches.
+ */
+public class DemoCachesLoadService implements Service {
+    /** Ignite instance. */
+    @IgniteInstanceResource
+    private Ignite ignite;
+
+    /** Thread pool to execute cache load operations. */
+    private ScheduledExecutorService cachePool;
+
+    /** */
+    private static final String COUNTRY_CACHE_NAME = "CountryCache";
+
+    /** */
+    private static final String DEPARTMENT_CACHE_NAME = "DepartmentCache";
+
+    /** */
+    private static final String EMPLOYEE_CACHE_NAME = "EmployeeCache";
+
+    /** */
+    private static final String PARKING_CACHE_NAME = "ParkingCache";
+
+    /** */
+    public static final String CAR_CACHE_NAME = "CarCache";
+
+    /** */
+    static final Set<String> DEMO_CACHES = new HashSet<>(Arrays.asList(COUNTRY_CACHE_NAME,
+        DEPARTMENT_CACHE_NAME, EMPLOYEE_CACHE_NAME, PARKING_CACHE_NAME, CAR_CACHE_NAME));
+
+    /** Countries count. */
+    private static final int CNTR_CNT = 10;
+
+    /** Departments count */
+    private static final int DEP_CNT = 100;
+
+    /** Employees count. */
+    private static final int EMPL_CNT = 1000;
+
+    /** Countries count. */
+    private static final int CAR_CNT = 100;
+
+    /** Departments count */
+    private static final int PARK_CNT = 10;
+
+    /** */
+    private static final Random rnd = new Random();
+
+    /** Maximum count read/write key. */
+    private final int cnt;
+
+    /** Time range in milliseconds. */
+    private final long range;
+
+    /**
+     * @param cnt Maximum count read/write key.
+     */
+    public DemoCachesLoadService(int cnt) {
+        this.cnt = cnt;
+
+        range = new java.util.Date().getTime();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        if (cachePool != null)
+            cachePool.shutdown();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        ignite.createCache(cacheCountry());
+        ignite.createCache(cacheDepartment());
+        ignite.createCache(cacheEmployee());
+        ignite.createCache(cacheCar());
+        ignite.createCache(cacheParking());
+
+        populateCacheEmployee();
+        populateCacheCar();
+
+        cachePool = AgentDemoUtils.newScheduledThreadPool(2, "demo-sql-load-cache-tasks");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        cachePool.scheduleWithFixedDelay(new Runnable() {
+            @Override public void run() {
+                try {
+                    IgniteCache<Integer, Employee> cacheEmployee = ignite.cache(EMPLOYEE_CACHE_NAME);
+
+                    if (cacheEmployee != null)
+                        try(Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                            for (int i = 0, n = 1; i < cnt; i++, n++) {
+                                Integer id = rnd.nextInt(EMPL_CNT);
+
+                                Integer depId = rnd.nextInt(DEP_CNT);
+
+                                double r = rnd.nextDouble();
+
+                                cacheEmployee.put(id, new Employee(id, depId, depId, "First name employee #" + n,
+                                    "Last name employee #" + n, "Email employee #" + n, "Phone number employee #" + n,
+                                    new java.sql.Date((long)(r * range)), "Job employee #" + n,
+                                    500 + AgentDemoUtils.round(r * 2000, 2)));
+
+                                if (rnd.nextBoolean())
+                                    cacheEmployee.remove(rnd.nextInt(EMPL_CNT));
+
+                                cacheEmployee.get(rnd.nextInt(EMPL_CNT));
+                            }
+
+                            if (rnd.nextInt(100) > 20)
+                                tx.commit();
+                        }
+                }
+                catch (Throwable e) {
+                    if (!e.getMessage().contains("cache is stopped"))
+                        ignite.log().error("Cache write task execution error", e);
+                }
+            }
+        }, 10, 3, TimeUnit.SECONDS);
+
+        cachePool.scheduleWithFixedDelay(new Runnable() {
+            @Override public void run() {
+                try {
+                    IgniteCache<Integer, Car> cache = ignite.cache(CAR_CACHE_NAME);
+
+                    if (cache != null)
+                        for (int i = 0; i < cnt; i++) {
+                            Integer carId = rnd.nextInt(CAR_CNT);
+
+                            cache.put(carId, new Car(carId, rnd.nextInt(PARK_CNT), "Car #" + (i + 1)));
+
+                            if (rnd.nextBoolean())
+                                cache.remove(rnd.nextInt(CAR_CNT));
+                        }
+                }
+                catch (IllegalStateException ignored) {
+                    // No-op.
+                }
+                catch (Throwable e) {
+                    if (!e.getMessage().contains("cache is stopped"))
+                        ignite.log().error("Cache write task execution error", e);
+                }
+            }
+        }, 10, 3, TimeUnit.SECONDS);
+    }
+
+
+    /**
+     * Create base cache configuration.
+     *
+     * @param name cache name.
+     * @return Cache configuration with basic properties set.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheConfiguration(String name) {
+        CacheConfiguration<K, V> ccfg = new CacheConfiguration<>(name);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg.setQueryDetailMetricsSize(10);
+        ccfg.setStartSize(100);
+        ccfg.setStatisticsEnabled(true);
+
+        return ccfg;
+    }
+
+    /**
+     * Configure cacheCountry.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheCountry() {
+        CacheConfiguration<K, V> ccfg = cacheConfiguration(COUNTRY_CACHE_NAME);
+
+        // Configure cacheCountry types.
+        Collection<QueryEntity> qryEntities = new ArrayList<>();
+
+        // COUNTRY.
+        QueryEntity type = new QueryEntity();
+
+        qryEntities.add(type);
+
+        type.setKeyType(Integer.class.getName());
+        type.setValueType(Country.class.getName());
+
+        // Query fields for COUNTRY.
+        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
+
+        qryFlds.put("id", "java.lang.Integer");
+        qryFlds.put("name", "java.lang.String");
+        qryFlds.put("population", "java.lang.Integer");
+
+        type.setFields(qryFlds);
+
+        ccfg.setQueryEntities(qryEntities);
+
+        return ccfg;
+    }
+
+    /**
+     * Configure cacheEmployee.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheDepartment() {
+        CacheConfiguration<K, V> ccfg = cacheConfiguration(DEPARTMENT_CACHE_NAME);
+
+        // Configure cacheDepartment types.
+        Collection<QueryEntity> qryEntities = new ArrayList<>();
+
+        // DEPARTMENT.
+        QueryEntity type = new QueryEntity();
+
+        qryEntities.add(type);
+
+        type.setKeyType(Integer.class.getName());
+        type.setValueType(Department.class.getName());
+
+        // Query fields for DEPARTMENT.
+        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
+
+        qryFlds.put("id", "java.lang.Integer");
+        qryFlds.put("countryId", "java.lang.Integer");
+        qryFlds.put("name", "java.lang.String");
+
+        type.setFields(qryFlds);
+
+        ccfg.setQueryEntities(qryEntities);
+
+        return ccfg;
+    }
+
+    /**
+     * Configure cacheEmployee.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheEmployee() {
+        CacheConfiguration<K, V> ccfg = cacheConfiguration(EMPLOYEE_CACHE_NAME);
+
+        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+        ccfg.setBackups(1);
+
+        // Configure cacheEmployee types.
+        Collection<QueryEntity> qryEntities = new ArrayList<>();
+
+        // EMPLOYEE.
+        QueryEntity type = new QueryEntity();
+
+        qryEntities.add(type);
+
+        type.setKeyType(Integer.class.getName());
+        type.setValueType(Employee.class.getName());
+
+        // Query fields for EMPLOYEE.
+        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
+
+        qryFlds.put("id", "java.lang.Integer");
+        qryFlds.put("departmentId", "java.lang.Integer");
+        qryFlds.put("managerId", "java.lang.Integer");
+        qryFlds.put("firstName", "java.lang.String");
+        qryFlds.put("lastName", "java.lang.String");
+        qryFlds.put("email", "java.lang.String");
+        qryFlds.put("phoneNumber", "java.lang.String");
+        qryFlds.put("hireDate", "java.sql.Date");
+        qryFlds.put("job", "java.lang.String");
+        qryFlds.put("salary", "java.lang.Double");
+
+        type.setFields(qryFlds);
+
+        // Indexes for EMPLOYEE.
+        Collection<QueryIndex> indexes = new ArrayList<>();
+
+        QueryIndex idx = new QueryIndex();
+
+        idx.setName("EMP_NAMES");
+        idx.setIndexType(QueryIndexType.SORTED);
+        LinkedHashMap<String, Boolean> indFlds = new LinkedHashMap<>();
+
+        indFlds.put("firstName", Boolean.FALSE);
+        indFlds.put("lastName", Boolean.FALSE);
+
+        idx.setFields(indFlds);
+
+        indexes.add(idx);
+        indexes.add(new QueryIndex("salary", QueryIndexType.SORTED, false, "EMP_SALARY"));
+
+        type.setIndexes(indexes);
+
+        ccfg.setQueryEntities(qryEntities);
+
+        return ccfg;
+    }
+
+    /**
+     * Configure cacheEmployee.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheParking() {
+        CacheConfiguration<K, V> ccfg = cacheConfiguration(PARKING_CACHE_NAME);
+
+        // Configure cacheParking types.
+        Collection<QueryEntity> qryEntities = new ArrayList<>();
+
+        // PARKING.
+        QueryEntity type = new QueryEntity();
+
+        qryEntities.add(type);
+
+        type.setKeyType(Integer.class.getName());
+        type.setValueType(Parking.class.getName());
+
+        // Query fields for PARKING.
+        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
+
+        qryFlds.put("id", "java.lang.Integer");
+        qryFlds.put("name", "java.lang.String");
+        qryFlds.put("capacity", "java.lang.Integer");
+
+        type.setFields(qryFlds);
+
+        ccfg.setQueryEntities(qryEntities);
+
+        return ccfg;
+    }
+
+    /**
+     * Configure cacheEmployee.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheCar() {
+        CacheConfiguration<K, V> ccfg = cacheConfiguration(CAR_CACHE_NAME);
+
+        // Configure cacheCar types.
+        Collection<QueryEntity> qryEntities = new ArrayList<>();
+
+        // CAR.
+        QueryEntity type = new QueryEntity();
+
+        qryEntities.add(type);
+
+        type.setKeyType(Integer.class.getName());
+        type.setValueType(Car.class.getName());
+
+        // Query fields for CAR.
+        LinkedHashMap<String, String> qryFlds = new LinkedHashMap<>();
+
+        qryFlds.put("id", "java.lang.Integer");
+        qryFlds.put("parkingId", "java.lang.Integer");
+        qryFlds.put("name", "java.lang.String");
+
+        type.setFields(qryFlds);
+
+        ccfg.setQueryEntities(qryEntities);
+
+        return ccfg;
+    }
+
+    /** */
+    private void populateCacheEmployee() {
+        if (ignite.log().isDebugEnabled())
+            ignite.log().debug("DEMO: Start employees population with data...");
+
+        IgniteCache<Integer, Country> cacheCountry = ignite.cache(COUNTRY_CACHE_NAME);
+
+        for (int i = 0, n = 1; i < CNTR_CNT; i++, n++)
+            cacheCountry.put(i, new Country(i, "Country #" + n, n * 10000000));
+
+        IgniteCache<Integer, Department> cacheDepartment = ignite.cache(DEPARTMENT_CACHE_NAME);
+
+        IgniteCache<Integer, Employee> cacheEmployee = ignite.cache(EMPLOYEE_CACHE_NAME);
+
+        for (int i = 0, n = 1; i < DEP_CNT; i++, n++) {
+            cacheDepartment.put(i, new Department(n, rnd.nextInt(CNTR_CNT), "Department #" + n));
+
+            double r = rnd.nextDouble();
+
+            cacheEmployee.put(i, new Employee(i, rnd.nextInt(DEP_CNT), null, "First name manager #" + n,
+                "Last name manager #" + n, "Email manager #" + n, "Phone number manager #" + n,
+                new java.sql.Date((long)(r * range)), "Job manager #" + n, 1000 + AgentDemoUtils.round(r * 4000, 2)));
+        }
+
+        for (int i = 0, n = 1; i < EMPL_CNT; i++, n++) {
+            Integer depId = rnd.nextInt(DEP_CNT);
+
+            double r = rnd.nextDouble();
+
+            cacheEmployee.put(i, new Employee(i, depId, depId, "First name employee #" + n,
+                "Last name employee #" + n, "Email employee #" + n, "Phone number employee #" + n,
+                new java.sql.Date((long)(r * range)), "Job employee #" + n, 500 + AgentDemoUtils.round(r * 2000, 2)));
+        }
+
+        if (ignite.log().isDebugEnabled())
+            ignite.log().debug("DEMO: Finished employees population.");
+    }
+
+    /** */
+    private void populateCacheCar() {
+        if (ignite.log().isDebugEnabled())
+            ignite.log().debug("DEMO: Start cars population...");
+
+        IgniteCache<Integer, Parking> cacheParking = ignite.cache(PARKING_CACHE_NAME);
+
+        for (int i = 0, n = 1; i < PARK_CNT; i++, n++)
+            cacheParking.put(i, new Parking(i, "Parking #" + n, n * 10));
+
+        IgniteCache<Integer, Car> cacheCar = ignite.cache(CAR_CACHE_NAME);
+
+        for (int i = 0, n = 1; i < CAR_CNT; i++, n++)
+            cacheCar.put(i, new Car(i, rnd.nextInt(PARK_CNT), "Car #" + n));
+
+        if (ignite.log().isDebugEnabled())
+            ignite.log().debug("DEMO: Finished cars population.");
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
new file mode 100644
index 0000000000000..57b26a2102ded
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import java.util.Random;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.console.demo.AgentDemoUtils;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Demo service. Create cache and populate it by random int pairs.
+ */
+public class DemoRandomCacheLoadService implements Service {
+    /** Ignite instance. */
+    @IgniteInstanceResource
+    private Ignite ignite;
+
+    /** Thread pool to execute cache load operations. */
+    private ScheduledExecutorService cachePool;
+
+    /** */
+    public static final String RANDOM_CACHE_NAME = "RandomCache";
+
+    /** Employees count. */
+    private static final int RND_CNT = 1024;
+
+    /** */
+    private static final Random rnd = new Random();
+
+    /** Maximum count read/write key. */
+    private final int cnt;
+
+    /**
+     * @param cnt Maximum count read/write key.
+     */
+    public DemoRandomCacheLoadService(int cnt) {
+        this.cnt = cnt;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        if (cachePool != null)
+            cachePool.shutdown();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        ignite.getOrCreateCache(cacheRandom());
+
+        cachePool = AgentDemoUtils.newScheduledThreadPool(2, "demo-sql-random-load-cache-tasks");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        cachePool.scheduleWithFixedDelay(new Runnable() {
+            @Override public void run() {
+                try {
+                    for (String cacheName : ignite.cacheNames()) {
+                        if (!DemoCachesLoadService.DEMO_CACHES.contains(cacheName)) {
+                            IgniteCache<Integer, Integer> cache = ignite.cache(cacheName);
+
+                            if (cache != null) {
+                                for (int i = 0, n = 1; i < cnt; i++, n++) {
+                                    Integer key = rnd.nextInt(RND_CNT);
+                                    Integer val = rnd.nextInt(RND_CNT);
+
+                                    cache.put(key, val);
+
+                                    if (rnd.nextInt(100) < 30)
+                                        cache.remove(key);
+                                }
+                            }
+                        }
+                    }
+                }
+                catch (Throwable e) {
+                    if (!e.getMessage().contains("cache is stopped"))
+                        ignite.log().error("Cache write task execution error", e);
+                }
+            }
+        }, 10, 3, TimeUnit.SECONDS);
+    }
+
+    /**
+     * Configure cacheCountry.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheRandom() {
+        CacheConfiguration<K, V> ccfg = new CacheConfiguration<>(RANDOM_CACHE_NAME);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg.setQueryDetailMetricsSize(10);
+        ccfg.setStartSize(100);
+        ccfg.setStatisticsEnabled(true);
+        ccfg.setIndexedTypes(Integer.class, Integer.class);
+
+        return ccfg;
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceClusterSingleton.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceClusterSingleton.java
new file mode 100644
index 0000000000000..8c0623a2901ba
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceClusterSingleton.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Demo service to provide on one node in cluster.
+ */
+public class DemoServiceClusterSingleton implements Service {
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceKeyAffinity.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceKeyAffinity.java
new file mode 100644
index 0000000000000..081ae27de88eb
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceKeyAffinity.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Demo service to provide for cache.
+ */
+public class DemoServiceKeyAffinity implements Service {
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceMultipleInstances.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceMultipleInstances.java
new file mode 100644
index 0000000000000..0d10753beb5ca
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceMultipleInstances.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Demo service to provide on all nodes.
+ */
+public class DemoServiceMultipleInstances implements Service {
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceNodeSingleton.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceNodeSingleton.java
new file mode 100644
index 0000000000000..4d491da90104d
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoServiceNodeSingleton.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.demo.service;
+
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Demo service to provide on all nodes by one.
+ */
+public class DemoServiceNodeSingleton implements Service {
+    /** {@inheritDoc} */
+    @Override public void cancel(ServiceContext ctx) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void init(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(ServiceContext ctx) throws Exception {
+        // No-op.
+    }
+}

From 2d6735a20ac8009d1ac3f003da4fcd319271bd71 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Thu, 9 Feb 2017 16:44:41 +0700
Subject: [PATCH 015/516] IGNITE-4676 Fixed hang if closure executed nested
 internal task with continuation. Added test.

(cherry picked from commit e7a5307)
---
 .../processors/job/GridJobWorker.java         |  4 +
 .../internal/GridContinuousTaskSelfTest.java  | 79 +++++++++++++++++++
 2 files changed, 83 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index 6a00d96a0abc2..acefde72eefab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -617,6 +617,10 @@ else if (X.hasCause(e, GridServiceNotFoundException.class) ||
                 // Finish here only if not held by this thread.
                 if (!HOLD.get())
                     finishJob(res, ex, sndRes);
+                else
+                    // Make sure flag is not set for current thread.
+                    // This may happen in case of nested internal task call with continuation.
+                    HOLD.set(false);
 
                 ctx.job().currentTaskSession(null);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
index 98e3c5afc8e2d..cec288714c100 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
@@ -21,10 +21,12 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.concurrent.Callable;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
@@ -43,7 +45,9 @@
 import org.apache.ignite.compute.ComputeTaskSessionAttributeListener;
 import org.apache.ignite.compute.ComputeTaskSessionFullSupport;
 import org.apache.ignite.compute.ComputeTaskSplitAdapter;
+import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.resources.TaskContinuousMapperResource;
@@ -51,6 +55,7 @@
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Continuous task test.
@@ -195,6 +200,80 @@ public void testMultipleHoldccCalls() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If test failed.
+     */
+    public void testClosureWithNestedInternalTask() throws Exception {
+        try {
+            IgniteEx ignite = startGrid(0);
+
+            ComputeTaskInternalFuture<String> fut = ignite.context().closure().callAsync(GridClosureCallMode.BALANCE, new Callable<String>() {
+                /** */
+                @IgniteInstanceResource
+                private IgniteEx g;
+
+                @Override public String call() throws Exception {
+                    return g.compute(g.cluster()).execute(NestedHoldccTask.class, null);
+                }
+            }, ignite.cluster().nodes());
+
+            assertEquals("DONE", fut.get(3000));
+        }
+        finally {
+            stopGrid(0, true);
+        }
+    }
+
+    /** Test task with continuation. */
+    @GridInternal
+    public static class NestedHoldccTask extends ComputeTaskAdapter<String, String> {
+        /** {@inheritDoc} */
+        @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
+            @Nullable String arg) throws IgniteException {
+            Map<ComputeJob, ClusterNode> map = new HashMap<>();
+
+            for (ClusterNode node : subgrid)
+                map.put(new NestedHoldccJob(), node);
+
+            return map;
+
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public String reduce(List<ComputeJobResult> results) throws IgniteException {
+            return results.get(0).getData();
+        }
+    }
+
+    /** Test job. */
+    public static class NestedHoldccJob extends ComputeJobAdapter {
+        /** */
+        @JobContextResource
+        private ComputeJobContext jobCtx;
+
+        /** */
+        private int cnt = 0;
+
+        /** {@inheritDoc} */
+        @Override public Object execute() throws IgniteException {
+            if (cnt < 1) {
+                cnt++;
+
+                jobCtx.holdcc();
+
+                new Timer().schedule(new TimerTask() {
+                    @Override public void run() {
+                        jobCtx.callcc();
+                    }
+                }, 500);
+
+                return "NOT DONE";
+            }
+
+            return "DONE";
+        }
+    }
+
     /** */
     @SuppressWarnings({"PublicInnerClass"})
     public static class TestMultipleHoldccCallsClosure implements IgniteClosure<Object, Boolean> {

From d307e2eced1fd10b007ee08c3dd113e7bb4f22ba Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Mon, 13 Feb 2017 17:35:29 +0700
Subject: [PATCH 016/516] IGNITE-4687 Added pool to process REST request in Web
 Agent.

(cherry picked from commit 262a341)
---
 .../ignite/console/agent/AgentLauncher.java   | 203 +++++++-------
 .../ignite/console/agent/AgentUtils.java      |  80 ++++++
 .../agent/handlers/AbstractHandler.java       | 110 --------
 .../agent/handlers/AbstractListener.java      | 104 +++++++
 ...baseHandler.java => DatabaseListener.java} | 264 ++++++++++--------
 .../{RestHandler.java => RestListener.java}   |  28 +-
 6 files changed, 441 insertions(+), 348 deletions(-)
 delete mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractHandler.java
 create mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
 rename modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/{DatabaseHandler.java => DatabaseListener.java} (57%)
 rename modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/{RestHandler.java => RestListener.java} (94%)

diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 049791f9107dc..a3d609fb395ae 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -41,8 +41,8 @@
 import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.X509TrustManager;
-import org.apache.ignite.console.agent.handlers.DatabaseHandler;
-import org.apache.ignite.console.agent.handlers.RestHandler;
+import org.apache.ignite.console.agent.handlers.DatabaseListener;
+import org.apache.ignite.console.agent.handlers.RestListener;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.log4j.Logger;
 import org.json.JSONException;
@@ -278,141 +278,138 @@ public static void main(String[] args) throws Exception {
             cfg.tokens(Arrays.asList(tokens.trim().split(",")));
         }
 
-        final RestHandler restHnd = new RestHandler(cfg);
+        URI uri = URI.create(cfg.serverUri());
 
-        try {
-            restHnd.start();
+        // Create proxy authenticator using passed properties.
+        switch (uri.getScheme()) {
+            case "http":
+            case "https":
+                final String username = System.getProperty(uri.getScheme() + ".proxyUsername");
+                final char[] pwd = System.getProperty(uri.getScheme() +  ".proxyPassword", "").toCharArray();
 
-            URI uri = URI.create(cfg.serverUri());
+                Authenticator.setDefault(new Authenticator() {
+                    @Override protected PasswordAuthentication getPasswordAuthentication() {
+                        return new PasswordAuthentication(username, pwd);
+                    }
+                });
 
-            // Create proxy authenticator using passed properties.
-            switch (uri.getScheme()) {
-                case "http":
-                case "https":
-                    final String username = System.getProperty(uri.getScheme() + ".proxyUsername");
-                    final char[] pwd = System.getProperty(uri.getScheme() +  ".proxyPassword", "").toCharArray();
+                break;
 
-                    Authenticator.setDefault(new Authenticator() {
-                        @Override protected PasswordAuthentication getPasswordAuthentication() {
-                            return new PasswordAuthentication(username, pwd);
-                        }
-                    });
-
-                    break;
+            default:
+                // No-op.
+        }
 
-                default:
-                    // No-op.
-            }
+        IO.Options opts = new IO.Options();
 
-            IO.Options opts = new IO.Options();
+        opts.path = "/agents";
 
-            opts.path = "/agents";
+        // Workaround for use self-signed certificate
+        if (Boolean.getBoolean("trust.all")) {
+            SSLContext ctx = SSLContext.getInstance("TLS");
 
-            // Workaround for use self-signed certificate
-            if (Boolean.getBoolean("trust.all")) {
-                SSLContext ctx = SSLContext.getInstance("TLS");
+            // Create an SSLContext that uses our TrustManager
+            ctx.init(null, getTrustManagers(), null);
 
-                // Create an SSLContext that uses our TrustManager
-                ctx.init(null, getTrustManagers(), null);
+            opts.sslContext = ctx;
+        }
 
-                opts.sslContext = ctx;
-            }
+        final Socket client = IO.socket(uri, opts);
 
-            final Socket client = IO.socket(uri, opts);
+        final RestListener restHnd = new RestListener(cfg);
 
-            try {
-                Emitter.Listener onConnecting = new Emitter.Listener() {
-                    @Override public void call(Object... args) {
-                        log.info("Connecting to: " + cfg.serverUri());
-                    }
-                };
+        final DatabaseListener dbHnd = new DatabaseListener(cfg);
 
-                Emitter.Listener onConnect = new Emitter.Listener() {
-                    @Override public void call(Object... args) {
-                        log.info("Connection established.");
+        try {
+            Emitter.Listener onConnecting = new Emitter.Listener() {
+                @Override public void call(Object... args) {
+                    log.info("Connecting to: " + cfg.serverUri());
+                }
+            };
 
-                        JSONObject authMsg = new JSONObject();
+            Emitter.Listener onConnect = new Emitter.Listener() {
+                @Override public void call(Object... args) {
+                    log.info("Connection established.");
 
-                        try {
-                            authMsg.put("tokens", cfg.tokens());
+                    JSONObject authMsg = new JSONObject();
 
-                            String clsName = AgentLauncher.class.getSimpleName() + ".class";
+                    try {
+                        authMsg.put("tokens", cfg.tokens());
 
-                            String clsPath = AgentLauncher.class.getResource(clsName).toString();
+                        String clsName = AgentLauncher.class.getSimpleName() + ".class";
 
-                            if (clsPath.startsWith("jar")) {
-                                String manifestPath = clsPath.substring(0, clsPath.lastIndexOf('!') + 1) +
-                                    "/META-INF/MANIFEST.MF";
+                        String clsPath = AgentLauncher.class.getResource(clsName).toString();
 
-                                Manifest manifest = new Manifest(new URL(manifestPath).openStream());
+                        if (clsPath.startsWith("jar")) {
+                            String manifestPath = clsPath.substring(0, clsPath.lastIndexOf('!') + 1) +
+                                "/META-INF/MANIFEST.MF";
 
-                                Attributes attr = manifest.getMainAttributes();
+                            Manifest manifest = new Manifest(new URL(manifestPath).openStream());
 
-                                authMsg.put("ver", attr.getValue("Implementation-Version"));
-                                authMsg.put("bt", attr.getValue("Build-Time"));
-                            }
+                            Attributes attr = manifest.getMainAttributes();
 
-                            client.emit("agent:auth", authMsg, new Ack() {
-                                @Override public void call(Object... args) {
-                                    // Authentication failed if response contains args.
-                                    if (args != null && args.length > 0) {
-                                        onDisconnect.call(args);
+                            authMsg.put("ver", attr.getValue("Implementation-Version"));
+                            authMsg.put("bt", attr.getValue("Build-Time"));
+                        }
 
-                                        System.exit(1);
-                                    }
+                        client.emit("agent:auth", authMsg, new Ack() {
+                            @Override public void call(Object... args) {
+                                // Authentication failed if response contains args.
+                                if (args != null && args.length > 0) {
+                                    onDisconnect.call(args);
 
-                                    log.info("Authentication success.");
+                                    System.exit(1);
                                 }
-                            });
-                        }
-                        catch (JSONException | IOException e) {
-                            log.error("Failed to construct authentication message", e);
 
-                            client.close();
-                        }
+                                log.info("Authentication success.");
+                            }
+                        });
                     }
-                };
-
-                DatabaseHandler dbHnd = new DatabaseHandler(cfg);
-
-                final CountDownLatch latch = new CountDownLatch(1);
-
-                client
-                    .on(EVENT_CONNECTING, onConnecting)
-                    .on(EVENT_CONNECT, onConnect)
-                    .on(EVENT_CONNECT_ERROR, onError)
-                    .on(EVENT_RECONNECTING, onConnecting)
-                    .on(EVENT_NODE_REST, restHnd)
-                    .on(EVENT_SCHEMA_IMPORT_DRIVERS, dbHnd.availableDriversListener())
-                    .on(EVENT_SCHEMA_IMPORT_SCHEMAS, dbHnd.schemasListener())
-                    .on(EVENT_SCHEMA_IMPORT_METADATA, dbHnd.metadataListener())
-                    .on(EVENT_ERROR, onError)
-                    .on(EVENT_DISCONNECT, onDisconnect)
-                    .on(EVENT_AGENT_WARNING, new Emitter.Listener() {
-                        @Override public void call(Object... args) {
-                            log.warn(args[0]);
-                        }
-                    })
-                    .on(EVENT_AGENT_CLOSE, new Emitter.Listener() {
-                        @Override public void call(Object... args) {
-                            onDisconnect.call(args);
+                    catch (JSONException | IOException e) {
+                        log.error("Failed to construct authentication message", e);
+
+                        client.close();
+                    }
+                }
+            };
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            client
+                .on(EVENT_CONNECTING, onConnecting)
+                .on(EVENT_CONNECT, onConnect)
+                .on(EVENT_CONNECT_ERROR, onError)
+                .on(EVENT_RECONNECTING, onConnecting)
+                .on(EVENT_NODE_REST, restHnd)
+                .on(EVENT_SCHEMA_IMPORT_DRIVERS, dbHnd.availableDriversListener())
+                .on(EVENT_SCHEMA_IMPORT_SCHEMAS, dbHnd.schemasListener())
+                .on(EVENT_SCHEMA_IMPORT_METADATA, dbHnd.metadataListener())
+                .on(EVENT_ERROR, onError)
+                .on(EVENT_DISCONNECT, onDisconnect)
+                .on(EVENT_AGENT_WARNING, new Emitter.Listener() {
+                    @Override public void call(Object... args) {
+                        log.warn(args[0]);
+                    }
+                })
+                .on(EVENT_AGENT_CLOSE, new Emitter.Listener() {
+                    @Override public void call(Object... args) {
+                        onDisconnect.call(args);
 
-                            client.off();
+                        client.off();
 
-                            latch.countDown();
-                        }
-                    });
+                        latch.countDown();
+                    }
+                });
 
-                client.connect();
+            client.connect();
 
-                latch.await();
-            }
-            finally {
-                client.close();
-            }
+            latch.await();
         }
         finally {
+            client.close();
+
             restHnd.stop();
+
+            dbHnd.stop();
         }
     }
 }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java
index 50a849adcc570..cb22651e9f83e 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java
@@ -17,11 +17,19 @@
 
 package org.apache.ignite.console.agent;
 
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
+import io.socket.client.Ack;
 import java.io.File;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.security.ProtectionDomain;
+import java.util.Arrays;
 import org.apache.log4j.Logger;
+import org.json.JSONArray;
+import org.json.JSONObject;
 
 /**
  * Utility methods.
@@ -30,6 +38,28 @@ public class AgentUtils {
     /** */
     private static final Logger log = Logger.getLogger(AgentUtils.class.getName());
 
+    /** JSON object mapper. */
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    /** */
+    private static final Ack NOOP_CB = new Ack() {
+        @Override public void call(Object... args) {
+            if (args != null && args.length > 0 && args[0] instanceof Throwable)
+                log.error("Failed to execute request on agent.", (Throwable) args[0]);
+            else
+                log.info("Request on agent successfully executed " + Arrays.toString(args));
+        }
+    };
+
+    static {
+        JsonOrgModule module = new JsonOrgModule();
+
+        mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
+        mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+
+        mapper.registerModule(module);
+    }
+    
     /**
      * Default constructor.
      */
@@ -108,4 +138,54 @@ public static File resolvePath(String path) {
 
         return null;
     }
+
+    /**
+     * Get callback from handler arguments.
+     *
+     * @param args Arguments.
+     * @return Callback or noop callback.
+     */
+    public static Ack safeCallback(Object[] args) {
+        boolean hasCb = args != null && args.length > 0 && args[args.length - 1] instanceof Ack;
+
+        return hasCb ? (Ack)args[args.length - 1] : NOOP_CB;
+    }
+
+    /**
+     * Remove callback from handler arguments.
+     * 
+     * @param args Arguments.
+     * @return Arguments without callback.
+     */
+    public static Object[] removeCallback(Object[] args) {
+        boolean hasCb = args != null && args.length > 0 && args[args.length - 1] instanceof Ack;
+
+        return hasCb ? Arrays.copyOf(args, args.length - 1) : args;
+    }
+
+    /**
+     * Map java object to JSON object.
+     * 
+     * @param obj Java object.
+     * @return {@link JSONObject} or {@link JSONArray}.
+     * @throws IllegalArgumentException If conversion fails due to incompatible type.
+     */
+    public static Object toJSON(Object obj) {
+        if (obj instanceof Iterable)
+            return mapper.convertValue(obj, JSONArray.class);
+
+        return mapper.convertValue(obj, JSONObject.class);
+    }
+
+    /**
+     * Map JSON object to java object.
+     *
+     * @param obj {@link JSONObject} or {@link JSONArray}.
+     * @param toValType Expected value type.
+     * @return Mapped object type of {@link T}.
+     * @throws IllegalArgumentException If conversion fails due to incompatible type.
+     */
+    public static <T> T fromJSON(Object obj, Class<T> toValType) throws IllegalArgumentException {
+        return mapper.convertValue(obj, toValType);
+    }
 }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractHandler.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractHandler.java
deleted file mode 100644
index 7e4e3204185ca..0000000000000
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractHandler.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.console.agent.handlers;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
-import io.socket.client.Ack;
-import io.socket.emitter.Emitter;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Map;
-import org.json.JSONArray;
-import org.json.JSONObject;
-
-/**
- * Base class for web socket handlers.
- */
-abstract class AbstractHandler implements Emitter.Listener {
-    /** JSON object mapper. */
-    private static final ObjectMapper mapper = new ObjectMapper();
-
-    static {
-        JsonOrgModule module = new JsonOrgModule();
-
-        mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
-        mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
-
-        mapper.registerModule(module);
-    }
-
-    /**
-     * @param obj Object.
-     * @return {@link JSONObject} or {@link JSONArray}.
-     */
-    private Object toJSON(Object obj) {
-        if (obj instanceof Iterable)
-            return mapper.convertValue(obj, JSONArray.class);
-
-        return mapper.convertValue(obj, JSONObject.class);
-    }
-
-    /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public final void call(Object... args) {
-        Ack cb = null;
-
-        try {
-            if (args == null || args.length == 0)
-                throw new IllegalArgumentException("Missing arguments.");
-
-            if (args.length > 2)
-                throw new IllegalArgumentException("Wrong arguments count, must be <= 2: " + Arrays.toString(args));
-
-            JSONObject lsnrArgs = null;
-
-            if (args.length == 1) {
-                if (args[0] instanceof JSONObject)
-                    lsnrArgs = (JSONObject)args[0];
-                else if (args[0] instanceof Ack)
-                    cb = (Ack)args[0];
-                else
-                    throw new IllegalArgumentException("Wrong type of argument, must be JSONObject or Ack: " + args[0]);
-            }
-            else {
-                if (args[0] != null && !(args[0] instanceof JSONObject))
-                    throw new IllegalArgumentException("Wrong type of argument, must be JSONObject: " + args[0]);
-
-                if (!(args[1] instanceof Ack))
-                    throw new IllegalArgumentException("Wrong type of argument, must be Ack: " + args[1]);
-
-                lsnrArgs = (JSONObject)args[0];
-
-                cb = (Ack)args[1];
-            }
-
-            Object res = execute(lsnrArgs == null ? Collections.emptyMap() : mapper.convertValue(lsnrArgs, Map.class));
-
-            if (cb != null)
-                cb.call(null, toJSON(res));
-        }
-        catch (Exception e) {
-            if (cb != null)
-                cb.call(e, null);
-        }
-    }
-
-    /**
-     * Execute command with specified arguments.
-     *
-     * @param args Map with method args.
-     */
-    public abstract Object execute(Map<String, Object> args) throws Exception;
-}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
new file mode 100644
index 0000000000000..987dac90bb77a
--- /dev/null
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.console.agent.handlers;
+
+import io.socket.client.Ack;
+import io.socket.emitter.Emitter;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.log4j.Logger;
+
+import static org.apache.ignite.console.agent.AgentUtils.removeCallback;
+import static org.apache.ignite.console.agent.AgentUtils.fromJSON;
+import static org.apache.ignite.console.agent.AgentUtils.safeCallback;
+import static org.apache.ignite.console.agent.AgentUtils.toJSON;
+
+/**
+ * Base class for web socket handlers.
+ */
+abstract class AbstractListener implements Emitter.Listener {
+    /** */
+    private ExecutorService pool;
+
+    /** */
+    final Logger log = Logger.getLogger(this.getClass().getName());
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override public final void call(Object... args) {
+        final Ack cb = safeCallback(args);
+
+        args = removeCallback(args);
+
+        try {
+            final Map<String, Object> params;
+
+            if (args == null || args.length == 0)
+                params = Collections.emptyMap();
+            else if (args.length == 1)
+                params = fromJSON(args[0], Map.class);
+            else
+                throw new IllegalArgumentException("Wrong arguments count, must be <= 1: " + Arrays.toString(args));
+
+            if (pool == null)
+                pool = newThreadPool();
+
+            pool.submit(new Runnable() {
+                @Override public void run() {
+                    try {
+                        Object res = execute(params);
+
+                        cb.call(null, toJSON(res));
+                    } catch (Exception e) {
+                        cb.call(e, null);
+                    }
+                }
+            });
+        }
+        catch (Exception e) {
+            cb.call(e, null);
+        }
+    }
+
+    /**
+     * Stop handler.
+     */
+    public void stop() {
+        if (pool != null)
+            pool.shutdownNow();
+    }
+
+    /**
+     * Creates a thread pool that can schedule commands to run after a given delay, or to execute periodically.
+     *
+     * @return Newly created thread pool.
+     */
+    protected ExecutorService newThreadPool() {
+        return Executors.newSingleThreadExecutor();
+    }
+
+    /**
+     * Execute command with specified arguments.
+     *
+     * @param args Map with method args.
+     */
+    public abstract Object execute(Map<String, Object> args) throws Exception;
+}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseHandler.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java
similarity index 57%
rename from modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseHandler.java
rename to modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java
index 02146d9a67996..4577228d2034f 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseHandler.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java
@@ -43,17 +43,138 @@
 /**
  * API to extract database metadata.
  */
-public class DatabaseHandler {
+public class DatabaseListener {
     /** */
-    private static final Logger log = Logger.getLogger(DatabaseHandler.class.getName());
+    private static final Logger log = Logger.getLogger(DatabaseListener.class.getName());
 
     /** */
     private final File driversFolder;
 
+    /** */
+    private final AbstractListener schemasLsnr = new AbstractListener() {
+        @Override public Object execute(Map<String, Object> args) throws Exception {
+            String driverPath = null;
+
+            if (args.containsKey("driverPath"))
+                driverPath = args.get("driverPath").toString();
+
+            if (!args.containsKey("driverClass"))
+                throw new IllegalArgumentException("Missing driverClass in arguments: " + args);
+
+            String driverCls = args.get("driverClass").toString();
+
+            if (!args.containsKey("url"))
+                throw new IllegalArgumentException("Missing url in arguments: " + args);
+
+            String url = args.get("url").toString();
+
+            if (!args.containsKey("info"))
+                throw new IllegalArgumentException("Missing info in arguments: " + args);
+
+            Properties info = new Properties();
+
+            info.putAll((Map)args.get("info"));
+
+            return schemas(driverPath, driverCls, url, info);
+        }
+    };
+
+    private final AbstractListener metadataLsnr = new AbstractListener() {
+        @SuppressWarnings("unchecked")
+        @Override public Object execute(Map<String, Object> args) throws Exception {
+            String driverPath = null;
+
+            if (args.containsKey("driverPath"))
+                driverPath = args.get("driverPath").toString();
+
+            if (!args.containsKey("driverClass"))
+                throw new IllegalArgumentException("Missing driverClass in arguments: " + args);
+
+            String driverCls = args.get("driverClass").toString();
+
+            if (!args.containsKey("url"))
+                throw new IllegalArgumentException("Missing url in arguments: " + args);
+
+            String url = args.get("url").toString();
+
+            if (!args.containsKey("info"))
+                throw new IllegalArgumentException("Missing info in arguments: " + args);
+
+            Properties info = new Properties();
+
+            info.putAll((Map)args.get("info"));
+
+            if (!args.containsKey("schemas"))
+                throw new IllegalArgumentException("Missing schemas in arguments: " + args);
+
+            List<String> schemas = (List<String>)args.get("schemas");
+
+            if (!args.containsKey("tablesOnly"))
+                throw new IllegalArgumentException("Missing tablesOnly in arguments: " + args);
+
+            boolean tblsOnly = (boolean)args.get("tablesOnly");
+
+            return metadata(driverPath, driverCls, url, info, schemas, tblsOnly);
+        }
+    };
+
+    private final AbstractListener availableDriversLsnr = new AbstractListener() {
+        @Override public Object execute(Map<String, Object> args) throws Exception {
+            if (driversFolder == null) {
+                log.info("JDBC drivers folder not specified, returning empty list");
+
+                return Collections.emptyList();
+            }
+
+            if (log.isDebugEnabled())
+                log.debug("Collecting JDBC drivers in folder: " + driversFolder.getPath());
+
+            File[] list = driversFolder.listFiles(new FilenameFilter() {
+                @Override public boolean accept(File dir, String name) {
+                    return name.endsWith(".jar");
+                }
+            });
+
+            if (list == null) {
+                log.info("JDBC drivers folder has no files, returning empty list");
+
+                return Collections.emptyList();
+            }
+
+            List<JdbcDriver> res = new ArrayList<>();
+
+            for (File file : list) {
+                try {
+                    boolean win = System.getProperty("os.name").contains("win");
+
+                    URL url = new URL("jar", null,
+                        "file:" + (win ? "/" : "") + file.getPath() + "!/META-INF/services/java.sql.Driver");
+
+                    try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) {
+                        String jdbcDriverCls = reader.readLine();
+
+                        res.add(new JdbcDriver(file.getName(), jdbcDriverCls));
+
+                        if (log.isDebugEnabled())
+                            log.debug("Found: [driver=" + file + ", class=" + jdbcDriverCls + "]");
+                    }
+                }
+                catch (IOException e) {
+                    res.add(new JdbcDriver(file.getName(), null));
+
+                    log.info("Found: [driver=" + file + "]");
+                    log.info("Failed to detect driver class: " + e.getMessage());
+                }
+            }
+
+            return res;
+        }
+    };
+
     /**
      * @param cfg Config.
      */
-    public DatabaseHandler(AgentConfiguration cfg) {
+    public DatabaseListener(AgentConfiguration cfg) {
         driversFolder = resolvePath(cfg.driversFolder() == null ? "jdbc-drivers" : cfg.driversFolder());
     }
 
@@ -105,39 +226,23 @@ protected Collection<String> schemas(String jdbcDriverJarPath, String jdbcDriver
         }
     }
 
+    /**
+     * Listener for drivers.
+     *
+     * @return Drivers in drivers folder
+     * @see AgentConfiguration#driversFolder
+     */
+    public Emitter.Listener availableDriversListener() {
+        return availableDriversLsnr;
+    }
+
     /**
      * Listener for schema names.
      *
      * @return Collection of schema names.
      */
     public Emitter.Listener schemasListener() {
-        return new AbstractHandler() {
-            @Override public Object execute(Map<String, Object> args) throws Exception {
-                String driverPath = null;
-
-                if (args.containsKey("driverPath"))
-                    driverPath = args.get("driverPath").toString();
-
-                if (!args.containsKey("driverClass"))
-                    throw new IllegalArgumentException("Missing driverClass in arguments: " + args);
-
-                String driverCls = args.get("driverClass").toString();
-
-                if (!args.containsKey("url"))
-                    throw new IllegalArgumentException("Missing url in arguments: " + args);
-
-                String url = args.get("url").toString();
-
-                if (!args.containsKey("info"))
-                    throw new IllegalArgumentException("Missing info in arguments: " + args);
-
-                Properties info = new Properties();
-
-                info.putAll((Map)args.get("info"));
-
-                return schemas(driverPath, driverCls, url, info);
-            }
-        };
+        return schemasLsnr;
     }
 
     /**
@@ -176,105 +281,18 @@ protected Collection<DbTable> metadata(String jdbcDriverJarPath, String jdbcDriv
      * @return Collection of tables.
      */
     public Emitter.Listener metadataListener() {
-        return new AbstractHandler() {
-            @SuppressWarnings("unchecked")
-            @Override public Object execute(Map<String, Object> args) throws Exception {
-                String driverPath = null;
-
-                if (args.containsKey("driverPath"))
-                    driverPath = args.get("driverPath").toString();
-
-                if (!args.containsKey("driverClass"))
-                    throw new IllegalArgumentException("Missing driverClass in arguments: " + args);
-
-                String driverCls = args.get("driverClass").toString();
-
-                if (!args.containsKey("url"))
-                    throw new IllegalArgumentException("Missing url in arguments: " + args);
-
-                String url = args.get("url").toString();
-
-                if (!args.containsKey("info"))
-                    throw new IllegalArgumentException("Missing info in arguments: " + args);
-
-                Properties info = new Properties();
-
-                info.putAll((Map)args.get("info"));
-
-                if (!args.containsKey("schemas"))
-                    throw new IllegalArgumentException("Missing schemas in arguments: " + args);
-
-                List<String> schemas = (List<String>)args.get("schemas");
-
-                if (!args.containsKey("tablesOnly"))
-                    throw new IllegalArgumentException("Missing tablesOnly in arguments: " + args);
-
-                boolean tblsOnly = (boolean)args.get("tablesOnly");
-
-                return metadata(driverPath, driverCls, url, info, schemas, tblsOnly);
-            }
-        };
+        return metadataLsnr;
     }
 
     /**
-     * Listener for drivers.
-     *
-     * @return Drivers in drivers folder
-     * @see AgentConfiguration#driversFolder
+     * Stop handler.
      */
-    public Emitter.Listener availableDriversListener() {
-        return new AbstractHandler() {
-            @Override public Object execute(Map<String, Object> args) throws Exception {
-                if (driversFolder == null) {
-                    log.info("JDBC drivers folder not specified, returning empty list");
-
-                    return Collections.emptyList();
-                }
-
-                if (log.isDebugEnabled())
-                    log.debug("Collecting JDBC drivers in folder: " + driversFolder.getPath());
-
-                File[] list = driversFolder.listFiles(new FilenameFilter() {
-                    @Override public boolean accept(File dir, String name) {
-                        return name.endsWith(".jar");
-                    }
-                });
+    public void stop() {
+        availableDriversLsnr.stop();
 
-                if (list == null) {
-                    log.info("JDBC drivers folder has no files, returning empty list");
+        schemasLsnr.stop();
 
-                    return Collections.emptyList();
-                }
-
-                List<JdbcDriver> res = new ArrayList<>();
-
-                for (File file : list) {
-                    try {
-                        boolean win = System.getProperty("os.name").contains("win");
-
-                        URL url = new URL("jar", null,
-                            "file:" + (win ? "/" : "") + file.getPath() + "!/META-INF/services/java.sql.Driver");
-
-                        try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) {
-                            String jdbcDriverCls = reader.readLine();
-
-                            res.add(new JdbcDriver(file.getName(), jdbcDriverCls));
-
-                            if (log.isDebugEnabled())
-                                log.debug("Found: [driver=" + file + ", class=" + jdbcDriverCls + "]");
-                        }
-                    }
-                    catch (IOException e) {
-                        res.add(new JdbcDriver(file.getName(), null));
-
-                        log.info("Found: [driver=" + file + "]");
-                        log.info("Failed to detect driver class: " + e.getMessage());
-                    }
-                }
-
-                return res;
-            }
-        };
+        metadataLsnr.stop();
     }
 
     /**
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestHandler.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
similarity index 94%
rename from modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestHandler.java
rename to modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
index 1b4b565ee68f0..1e86549e25927 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestHandler.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
@@ -24,6 +24,8 @@
 import java.nio.charset.Charset;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import org.apache.commons.codec.Charsets;
 import org.apache.http.Header;
 import org.apache.http.NameValuePair;
@@ -45,9 +47,9 @@
 /**
  * API to translate REST requests to Ignite cluster.
  */
-public class RestHandler extends AbstractHandler {
+public class RestListener extends AbstractListener {
     /** */
-    private static final Logger log = Logger.getLogger(RestHandler.class.getName());
+    private static final Logger log = Logger.getLogger(RestListener.class.getName());
 
     /** */
     private final AgentConfiguration cfg;
@@ -58,21 +60,18 @@ public class RestHandler extends AbstractHandler {
     /**
      * @param cfg Config.
      */
-    public RestHandler(AgentConfiguration cfg) {
+    public RestListener(AgentConfiguration cfg) {
+        super();
+
         this.cfg = cfg;
-    }
 
-    /**
-     * Start HTTP client for communication with node via REST.
-     */
-    public void start() {
         httpClient = HttpClientBuilder.create().build();
     }
 
-    /**
-     * Stop HTTP client.
-     */
-    public void stop() {
+    /** {@inheritDoc} */
+    @Override public void stop() {
+        super.stop();
+
         if (httpClient != null) {
             try {
                 httpClient.close();
@@ -83,6 +82,11 @@ public void stop() {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override protected ExecutorService newThreadPool() {
+        return Executors.newCachedThreadPool();
+    }
+
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public Object execute(Map<String, Object> args) throws Exception {

From 8874f99f44dc2edf08a525619edb49d5db70b938 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 14 Feb 2017 18:44:57 +0300
Subject: [PATCH 017/516] IGNITE-4641 - Refresh client attributes during
 reconnect

---
 .../internal/managers/GridManagerAdapter.java |   4 +
 .../apache/ignite/spi/IgniteSpiAdapter.java   |   5 +
 .../apache/ignite/spi/IgniteSpiContext.java   |   6 +
 .../ignite/spi/discovery/tcp/ClientImpl.java  |   2 +-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   5 +
 .../tcp/internal/TcpDiscoveryNode.java        |   6 +-
 ...ryNodeAttributesUpdateOnReconnectTest.java | 110 +++++++++++++++++
 .../tcp/TestReconnectPluginProvider.java      | 111 ++++++++++++++++++
 .../discovery/tcp/TestReconnectProcessor.java |  93 +++++++++++++++
 .../testframework/GridSpiTestContext.java     |   5 +
 .../IgniteSpiDiscoverySelfTestSuite.java      |   3 +
 .../org.apache.ignite.plugin.PluginProvider   |   1 +
 parent/pom.xml                                |   1 +
 13 files changed, 349 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryNodeAttributesUpdateOnReconnectTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectPluginProvider.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
 create mode 100644 modules/core/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
index 584cc56e23b3f..25cc715ee12b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
@@ -569,6 +569,10 @@ protected final String stopInfo() {
                         ctx.timeout().removeTimeoutObject(new GridSpiTimeoutObject(obj));
                     }
 
+                    @Override public Map<String, Object> nodeAttributes() {
+                        return ctx.nodeAttributes();
+                    }
+
                     /**
                      * @param e Exception to handle.
                      * @return GridSpiException Converted exception.
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index 219d07be5f725..8879364dca5dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -928,5 +928,10 @@ private class GridDummySpiContext implements IgniteSpiContext {
 
             ((IgniteKernal)ignite0).context().timeout().removeTimeoutObject(new GridSpiTimeoutObject(obj));
         }
+
+        /** {@inheritDoc} */
+        @Override public Map<String, Object> nodeAttributes() {
+            return Collections.emptyMap();
+        }
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
index 5eb5227a0a46c..96b3e61a94131 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
@@ -19,6 +19,7 @@
 
 import java.io.Serializable;
 import java.util.Collection;
+import java.util.Map;
 import java.util.UUID;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteException;
@@ -352,4 +353,9 @@ public interface IgniteSpiContext {
      * @param c Timeout object.
      */
     public void removeTimeoutObject(IgniteSpiTimeoutObject c);
+
+    /**
+     * @return Current node attributes.
+     */
+    public Map<String, Object> nodeAttributes();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 39c539c30fb89..932e7d1e754a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -628,7 +628,7 @@ else if (addrs.isEmpty()) {
                     TcpDiscoveryNode node = locNode;
 
                     if (locNode.order() > 0)
-                        node = locNode.clientReconnectNode();
+                        node = locNode.clientReconnectNode(spi.spiCtx.nodeAttributes());
 
                     msg = new TcpDiscoveryJoinRequestMessage(node, spi.collectExchangeData(getLocalNodeId()));
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 45933e112f78d..00ae97d1e302c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -409,6 +409,9 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, T
     /** */
     private boolean clientReconnectDisabled;
 
+    /** */
+    protected IgniteSpiContext spiCtx;
+
     /** {@inheritDoc} */
     @Override public String getSpiState() {
         return impl.getSpiState();
@@ -1161,6 +1164,8 @@ LinkedHashSet<InetSocketAddress> getNodeAddresses(TcpDiscoveryNode node, boolean
     @Override protected void onContextInitialized0(IgniteSpiContext spiCtx) throws IgniteSpiException {
         super.onContextInitialized0(spiCtx);
 
+        this.spiCtx = spiCtx;
+
         ctxInitLatch.countDown();
 
         ipFinder.onSpiContextInitialized(spiCtx);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
index 307aefe641908..d8b1fc12243e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
@@ -26,6 +26,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -513,13 +514,14 @@ public void onClientDisconnected(UUID newId) {
     }
 
     /**
+     * @param nodeAttrs Current node attributes.
      * @return Copy of local node for client reconnect request.
      */
-    public TcpDiscoveryNode clientReconnectNode() {
+    public TcpDiscoveryNode clientReconnectNode(Map<String, Object> nodeAttrs) {
         TcpDiscoveryNode node = new TcpDiscoveryNode(id, addrs, hostNames, discPort, metricsProvider, ver,
             null);
 
-        node.attrs = attrs;
+        node.attrs = Collections.unmodifiableMap(new HashMap<>(nodeAttrs));
         node.clientRouterNodeId = clientRouterNodeId;
 
         return node;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryNodeAttributesUpdateOnReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryNodeAttributesUpdateOnReconnectTest.java
new file mode 100644
index 0000000000000..56dc4ece5f4fc
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryNodeAttributesUpdateOnReconnectTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteEvents;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.internal.IgniteClientReconnectAbstractTest.reconnectClientNode;
+
+/**
+ * Checks whether on client reconnect node attributes from kernal context are sent.
+ */
+public class TcpDiscoveryNodeAttributesUpdateOnReconnectTest extends GridCommonAbstractTest {
+    /** */
+    private volatile String rejoinAttr;
+
+    /** */
+    @LoggerResource
+    private IgniteLogger log;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("client")) {
+            Map<String, String> attrs = new HashMap<>();
+
+            attrs.put("test", "1");
+
+            cfg.setUserAttributes(attrs);
+            cfg.setClientMode(true);
+        }
+
+        IgniteClientReconnectAbstractTest.TestTcpDiscoverySpi spi = new IgniteClientReconnectAbstractTest.TestTcpDiscoverySpi();
+
+        TcpDiscoveryIpFinder finder = ((TcpDiscoverySpi)cfg.getDiscoverySpi()).getIpFinder();
+
+        spi.setIpFinder(finder);
+
+        cfg.setDiscoverySpi(spi);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        TestReconnectPluginProvider.enabled = false;
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        TestReconnectPluginProvider.enabled = true;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnect() throws Exception {
+        Ignite srv = startGrid("server");
+
+        IgniteEvents evts = srv.events();
+
+        evts.enableLocal(EventType.EVTS_DISCOVERY_ALL);
+        evts.localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                ClusterNode node = ((DiscoveryEvent)evt).eventNode();
+
+                rejoinAttr = node.attribute("test");
+
+                return true;
+            }
+        }, EventType.EVT_NODE_JOINED);
+
+        Ignite client = startGrid("client");
+
+        reconnectClientNode(log, client, srv, null);
+
+        assertEquals("2", rejoinAttr);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectPluginProvider.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectPluginProvider.java
new file mode 100644
index 0000000000000..692774ca79dc8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectPluginProvider.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.Serializable;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
+import org.apache.ignite.plugin.ExtensionRegistry;
+import org.apache.ignite.plugin.IgnitePlugin;
+import org.apache.ignite.plugin.PluginContext;
+import org.apache.ignite.plugin.PluginProvider;
+import org.apache.ignite.plugin.PluginValidationException;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Creates TestReconnectProcessor.
+ */
+public class TestReconnectPluginProvider implements PluginProvider {
+    /** */
+    private GridKernalContext igniteCtx;
+
+    /** */
+    public static volatile boolean enabled;
+
+    /** {@inheritDoc} */
+    @Override public String name() {
+        return "TestReconnectPlugin";
+    }
+
+    /** {@inheritDoc} */
+    @Override public String version() {
+        return "1.0";
+    }
+
+    /** {@inheritDoc} */
+    @Override public String copyright() {
+        return "";
+    }
+
+    /** {@inheritDoc} */
+    @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) {
+        igniteCtx = ((IgniteKernal)ctx.grid()).context();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void start(PluginContext ctx) throws IgniteCheckedException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void stop(boolean cancel) throws IgniteCheckedException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onIgniteStart() throws IgniteCheckedException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onIgniteStop(boolean cancel) {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Serializable provideDiscoveryData(UUID nodeId) {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void receiveDiscoveryData(UUID nodeId, Serializable data) {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void validateNewNode(ClusterNode node) throws PluginValidationException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object createComponent(PluginContext ctx, Class cls) {
+        if (enabled && GridSecurityProcessor.class.equals(cls))
+            return new TestReconnectProcessor(igniteCtx);
+
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgnitePlugin plugin() {
+        return new IgnitePlugin() {};
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
new file mode 100644
index 0000000000000..f0ed35c4ccfab
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.Collection;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.plugin.security.AuthenticationContext;
+import org.apache.ignite.plugin.security.SecurityCredentials;
+import org.apache.ignite.plugin.security.SecurityException;
+import org.apache.ignite.plugin.security.SecurityPermission;
+import org.apache.ignite.plugin.security.SecuritySubject;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Updates node attributes on disconnect.
+ */
+public class TestReconnectProcessor extends GridProcessorAdapter implements GridSecurityProcessor {
+    /**
+     * @param ctx Kernal context.
+     */
+    protected TestReconnectProcessor(GridKernalContext ctx) {
+        super(ctx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public SecurityContext authenticateNode(ClusterNode node,
+        SecurityCredentials cred) throws IgniteCheckedException {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isGlobalNodeAuthentication() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public SecurityContext authenticate(AuthenticationContext ctx) throws IgniteCheckedException {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<SecuritySubject> authenticatedSubjects() throws IgniteCheckedException {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public SecuritySubject authenticatedSubject(UUID subjId) throws IgniteCheckedException {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void authorize(String name, SecurityPermission perm,
+        @Nullable SecurityContext securityCtx) throws SecurityException {
+
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onSessionExpired(UUID subjId) {
+
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean enabled() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onDisconnected(IgniteFuture<?> reconnectFut) throws IgniteCheckedException {
+        ctx.addNodeAttribute("test", "2");
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index ac50ef90e29e1..1c8acbc3960ce 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -601,6 +601,11 @@ public void triggerEvent(Event evt) {
             timeoutProcessor.removeTimeoutObject(new GridSpiTimeoutObject(obj));
     }
 
+    /** {@inheritDoc} */
+    @Override public Map<String, Object> nodeAttributes() {
+        return Collections.emptyMap();
+    }
+
     /**
      * @param cacheName Cache name.
      * @return Map representing cache.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 5f870a474578b..548e1a5a51f81 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMultiThreadedTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeAttributesUpdateOnReconnectTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeConfigConsistentIdSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeConsistentIdSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryRestartTest;
@@ -84,6 +85,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(TcpDiscoveryRestartTest.class));
         suite.addTest(new TestSuite(TcpDiscoveryMultiThreadedTest.class));
 
+        suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
+
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
 
diff --git a/modules/core/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider b/modules/core/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider
new file mode 100644
index 0000000000000..a7fdf4384e626
--- /dev/null
+++ b/modules/core/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider
@@ -0,0 +1 @@
+org.apache.ignite.spi.discovery.tcp.TestReconnectPluginProvider
\ No newline at end of file
diff --git a/parent/pom.xml b/parent/pom.xml
index 88532a70dfb5a..8bf5dde9e12a8 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -821,6 +821,7 @@
                                         <exclude>src/main/java/META-INF/services/javax.cache.spi.CachingProvider</exclude><!--cannot be changed-->
                                         <exclude>src/main/java/org/jetbrains/annotations/*.java</exclude><!--copyright-->
                                         <exclude>src/main/resources/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider</exclude><!--cannot be changed-->
+                                        <exclude>/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider</exclude>
                                         <exclude>dev-tools/IGNITE-*.patch</exclude>
                                         <exclude>dev-tools/.gradle/**/*</exclude>
                                         <exclude>dev-tools/gradle/wrapper/**/*</exclude>

From ca5956bedfc0c3bd18290c64b6a6c2e3f114a440 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Tue, 14 Feb 2017 23:28:06 +0700
Subject: [PATCH 018/516] IGNITE-4472 UI fix, minor fixes

(cherry picked from commit 79e1e53)
---
 .../web-console/backend/routes/activities.js  |   7 -
 .../backend/services/activities.js            |  27 +--
 .../test/unit/ActivitiesService.test.js       | 131 +++++++++++++
 .../web-console/frontend/app/app.config.js    |   4 +
 .../activities-user-dialog.controller.js      |  39 +---
 .../activities-user-dialog.jade               |   2 +-
 .../activities-user-dialog/index.js           |   5 +-
 .../form-field-datepicker.jade                |   4 +-
 .../list-of-registered-users.column-defs.js   |  26 +--
 .../list-of-registered-users.controller.js    |  32 ++-
 .../app/core/activities/Activities.data.js    |   5 -
 modules/web-console/frontend/app/data/i18n.js |   1 +
 .../ui-ace-pom/ui-ace-pom.controller.js       |   4 +-
 .../app/modules/agent/agent.module.js         |  15 --
 .../modules/configuration/Version.service.js  |  13 +-
 .../configuration/generator/Maven.service.js  |  10 +-
 .../configuration/summary/summary.worker.js   |   6 +-
 modules/web-console/frontend/package.json     | 182 +++++++++---------
 .../frontend/public/stylesheets/style.scss    |   8 -
 .../public/stylesheets/variables.scss         |   1 -
 .../views/templates/agent-download.jade       |   6 +-
 21 files changed, 290 insertions(+), 238 deletions(-)
 create mode 100644 modules/web-console/backend/test/unit/ActivitiesService.test.js

diff --git a/modules/web-console/backend/routes/activities.js b/modules/web-console/backend/routes/activities.js
index 08c27cf1cdf8c..ad0e46962821d 100644
--- a/modules/web-console/backend/routes/activities.js
+++ b/modules/web-console/backend/routes/activities.js
@@ -33,13 +33,6 @@ module.exports.factory = function(express, activitiesService) {
     return new Promise((factoryResolve) => {
         const router = new express.Router();
 
-        // Get user activities.
-        router.get('/user/:userId', (req, res) => {
-            activitiesService.listByUser(req.params.userId, req.query)
-                .then(res.api.ok)
-                .catch(res.api.error);
-        });
-
         // Post user activities to page.
         router.post('/page', (req, res) => {
             activitiesService.merge(req.user._id, req.body)
diff --git a/modules/web-console/backend/services/activities.js b/modules/web-console/backend/services/activities.js
index 7f3a7775f83d2..124c775dc7a86 100644
--- a/modules/web-console/backend/services/activities.js
+++ b/modules/web-console/backend/services/activities.js
@@ -35,10 +35,12 @@ module.exports.factory = (_, mongo) => {
          * Update page activities.
          *
          * @param {String} owner - User ID
-         * @param {Object} page - The page
+         * @param {String} action - Action string presentation.
+         * @param {String} group - Action group string presentation.
+         * @param {Date} [date] - Optional date to save in activity.
          * @returns {Promise.<mongo.ObjectId>} that resolve activity
          */
-        static merge(owner, {action, group}) {
+        static merge(owner, {action, group}, date = new Date()) {
             mongo.Account.findById(owner)
                 .then((user) => {
                     user.lastActivity = new Date();
@@ -46,8 +48,6 @@ module.exports.factory = (_, mongo) => {
                     return user.save();
                 });
 
-            const date = new Date();
-
             date.setDate(1);
             date.setHours(0, 0, 0, 0);
 
@@ -63,25 +63,6 @@ module.exports.factory = (_, mongo) => {
                 });
         }
 
-        /**
-         * Get user activities
-         * @param {String} owner - User ID
-         * @returns {Promise.<mongo.ObjectId>} that resolve activities
-         */
-        static listByUser(owner, {startDate, endDate}) {
-            const $match = {owner};
-
-            if (startDate)
-                $match.date = {$gte: new Date(startDate)};
-
-            if (endDate) {
-                $match.date = $match.date || {};
-                $match.date.$lt = new Date(endDate);
-            }
-
-            return mongo.Activities.find($match);
-        }
-
         static total({startDate, endDate}) {
             const $match = {};
 
diff --git a/modules/web-console/backend/test/unit/ActivitiesService.test.js b/modules/web-console/backend/test/unit/ActivitiesService.test.js
new file mode 100644
index 0000000000000..40088bf75c333
--- /dev/null
+++ b/modules/web-console/backend/test/unit/ActivitiesService.test.js
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const assert = require('chai').assert;
+const injector = require('../injector');
+const testAccounts = require('../data/accounts.json');
+
+let activitiesService;
+let mongo;
+let db;
+
+const owner = testAccounts[0]._id;
+const group = 'test';
+const action1 = '/test/activity1';
+const action2 = '/test/activity2';
+
+suite('ActivitiesServiceTestsSuite', () => {
+    suiteSetup(() => {
+        return Promise.all([
+            injector('services/activities'),
+            injector('mongo'),
+            injector('dbHelper')
+        ])
+            .then(([_activitiesService, _mongo, _db]) => {
+                mongo = _mongo;
+                activitiesService = _activitiesService;
+                db = _db;
+            });
+    });
+
+    setup(() => db.init());
+
+    test('Activities creation and update', (done) => {
+        activitiesService.merge(owner, { group, action: action1 })
+            .then((activity) => {
+                assert.isNotNull(activity);
+                assert.equal(activity.amount, 1);
+
+                return mongo.Activities.findById(activity._id);
+            })
+            .then((activityDoc) => {
+                assert.isNotNull(activityDoc);
+                assert.equal(activityDoc.amount, 1);
+            })
+            .then(() => activitiesService.merge(owner, { group, action: action1 }))
+            .then((activity) => {
+                assert.isNotNull(activity);
+                assert.equal(activity.amount, 2);
+
+                return mongo.Activities.findById(activity._id);
+            })
+            .then((activityDoc) => {
+                assert.isNotNull(activityDoc);
+                assert.equal(activityDoc.amount, 2);
+            })
+            .then(done)
+            .catch(done);
+    });
+
+    test('Activities total and detail information', (done) => {
+        const startDate = new Date();
+
+        startDate.setDate(1);
+        startDate.setHours(0, 0, 0, 0);
+
+        const endDate = new Date(startDate);
+        endDate.setMonth(endDate.getMonth() + 1);
+
+        Promise.all([
+            activitiesService.merge(owner, {group, action: action1}),
+            activitiesService.merge(owner, {group, action: action2})
+        ])
+            .then(() => activitiesService.total(owner, {startDate, endDate}))
+            .then((activities) =>
+                assert.equal(activities[owner].test, 2)
+            )
+            .then(() => activitiesService.detail(owner, {startDate, endDate}))
+            .then((activities) =>
+                assert.deepEqual(activities[owner], {
+                    '/test/activity2': 1, '/test/activity1': 1
+                })
+            )
+            .then(done)
+            .catch(done);
+    });
+
+    test('Activities periods', (done) => {
+        const startDate = new Date();
+
+        startDate.setDate(1);
+        startDate.setHours(0, 0, 0, 0);
+
+        const nextMonth = (baseDate) => {
+            const date = new Date(baseDate);
+
+            date.setMonth(date.getMonth() + 1);
+
+            return date;
+        };
+
+        const borderDate = nextMonth(startDate);
+        const endDate = nextMonth(borderDate);
+
+        activitiesService.merge(owner, { group, action: action1 })
+            .then(() => activitiesService.merge(owner, { group, action: action1 }, borderDate))
+            .then(() => activitiesService.total({ startDate, endDate: borderDate }))
+            .then((activities) =>
+                assert.equal(activities[owner].test, 1)
+            )
+            .then(() => activitiesService.total({ startDate: borderDate, endDate }))
+            .then((activities) =>
+                assert.equal(activities[owner].test, 1)
+            )
+            .then(done)
+            .catch(done);
+    });
+});
diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js
index 0e85711141799..39d761f418eef 100644
--- a/modules/web-console/frontend/app/app.config.js
+++ b/modules/web-console/frontend/app/app.config.js
@@ -103,3 +103,7 @@ igniteConsoleCfg.config(['$datepickerProvider', ($datepickerProvider) => {
         iconRight: 'icon-datepicker-right'
     });
 }]);
+
+igniteConsoleCfg.config(['$translateProvider', ($translateProvider) => {
+    $translateProvider.useSanitizeValueStrategy('sanitize');
+}]);
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
index 46853b262eaad..078f725fd8e2c 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
@@ -15,46 +15,13 @@
  * limitations under the License.
  */
 
-const COLUMNS_DEFS = [
-    {displayName: 'Action', field: 'action', minWidth: 65 },
-    {displayName: 'Description', field: 'title', minWidth: 65 },
-    {displayName: 'Visited', field: 'amount', minWidth: 65 }
-];
-
 export default class ActivitiesCtrl {
-    static $inject = ['$state', 'user', 'params', 'IgniteActivitiesData'];
+    static $inject = ['user'];
 
-    constructor($state, user, params, ActivitiesData) {
+    constructor(user) {
         const $ctrl = this;
-        const userId = user._id;
 
         $ctrl.user = user;
-
-        $ctrl.gridOptions = {
-            data: [],
-            columnVirtualizationThreshold: 30,
-            columnDefs: COLUMNS_DEFS,
-            categories: [
-                {name: 'Action', visible: true, selectable: true},
-                {name: 'Description', visible: true, selectable: true},
-                {name: 'Visited', visible: true, selectable: true}
-            ],
-            enableRowSelection: false,
-            enableRowHeaderSelection: false,
-            enableColumnMenus: false,
-            multiSelect: false,
-            modifierKeysToMultiSelect: true,
-            noUnselect: true,
-            flatEntityAccess: true,
-            fastWatch: true,
-            onRegisterApi: (api) => {
-                $ctrl.gridApi = api;
-            }
-        };
-
-        ActivitiesData.listByUser(userId, params)
-            .then((data) => {
-                $ctrl.data = data;
-            });
+        $ctrl.data = _.map(user.activitiesDetail, (amount, action) => ({ action, amount }));
     }
 }
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
index 2c55ebd6da29a..074851ca55d66 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
@@ -24,7 +24,7 @@
             .modal-body.modal-body-with-scroll(id='activities-user-dialog')
                 table.table.table-striped.table-bordered.table-hover(scrollable-container='#activities-user-dialog' st-table='displayedRows' st-safe-src='ctrl.data')
                     thead
-                        th.text-center(st-sort='title') Description
+                        th.text-center(st-sort='action | translate') Description
                         th.text-center(st-sort='action') Action
                         th.text-center(st-sort='amount') Visited
                     tbody
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/index.js b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
index 03d35852b147a..dca6ba92ede6d 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/index.js
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
@@ -18,13 +18,12 @@
  import controller from './activities-user-dialog.controller';
  import templateUrl from './activities-user-dialog.jade';
 
- export default ['$modal', ($modal) => ({ show = true, user, params }) => {
+ export default ['$modal', ($modal) => ({ show = true, user }) => {
      const ActivitiesUserDialog = $modal({
          templateUrl,
          show,
          resolve: {
-             user: () => user,
-             params: () => params
+             user: () => user
          },
          placement: 'center',
          controller,
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
index 6792977b65831..2578cf4b17189 100644
--- a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
+++ b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
@@ -28,8 +28,8 @@ mixin ignite-form-field-datepicker(label, model, name, disabled, required, place
             data-ng-disabled=disabled && '#{disabled}'
 
             bs-datepicker
-            data-date-format='MMM yyyy' 
-            data-start-view='1' 
+            data-date-format='MMM yyyy'
+            data-start-view='1'
             data-min-view='1' 
             data-max-date='today'
 
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
index 61e1bd8575b22..4dc465505b6ab 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
@@ -49,7 +49,7 @@ const ACTIONS_TEMPLATE = `
 const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
 
 export default [
-    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'test', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, pinnedLeft: true},
+    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, pinnedLeft: true},
     {displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by name...' }, pinnedLeft: true},
     {displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
     {displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 160, enableFiltering: true},
@@ -62,19 +62,19 @@ export default [
     {displayName: 'Caches count', categoryDisplayName: 'Configurations', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: 'counters.caches', type: 'number', headerTooltip: 'Caches count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
     {displayName: 'IGFS count', categoryDisplayName: 'Configurations', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: 'counters.igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
     // Activities Total
-    {displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Configuration', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Queries', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Demo', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'AD', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Agent Download', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'AS', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Agent Start', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Total count of configuration usages', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Total count of queries usages', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Total count of demo startup', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Total count of agent downloads', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Str', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Total count of agent startup', minWidth: 50, width: 50, enableFiltering: false},
     // Activities Configuration
-    {displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration Clusters', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration Model', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', headerTooltip: 'Configuration Caches', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration clusters', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration model', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', headerTooltip: 'Configuration caches', minWidth: 50, width: 80, enableFiltering: false, visible: false},
     {displayName: 'IGFS', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/igfs"] || 0', type: 'number', headerTooltip: 'Configuration IGFS', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', headerTooltip: 'Configuration Summary', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', headerTooltip: 'Configuration summary', minWidth: 50, width: 80, enableFiltering: false, visible: false},
     // Activities Queries
-    {displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', headerTooltip: 'Query execute', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', headerTooltip: 'Query explain', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', headerTooltip: 'Scan', minWidth: 50, width: 80, enableFiltering: false, visible: false}
+    {displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', headerTooltip: 'Query executions', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', headerTooltip: 'Query explain executions', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', headerTooltip: 'Scan query executions', minWidth: 50, width: 80, enableFiltering: false, visible: false}
 ];
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 19f7921ac7f17..1f2a348daad7a 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -20,15 +20,26 @@ import headerTemplate from 'app/components/ui-grid-header/ui-grid-header.jade';
 import columnDefs from './list-of-registered-users.column-defs';
 import categories from './list-of-registered-users.categories';
 
+const rowTemplate = `<div
+  ng-repeat="(colRenderIndex, col) in colContainer.renderedColumns track by col.uid"
+  ng-mouseover="grid.api.selection.selectRow(row.entity);"
+  ui-grid-one-bind-id-grid="rowRenderIndex + '-' + col.uid + '-cell'"
+  class="ui-grid-cell"
+  ng-class="{ 'ui-grid-row-header-cell': col.isRowHeader }"
+  role="{{col.isRowHeader ? 'rowheader' : 'gridcell'}}"
+  ui-grid-cell/>`;
+
 export default class IgniteListOfRegisteredUsersCtrl {
-    static $inject = ['$scope', '$state', '$templateCache', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
+    static $inject = ['$scope', '$state', '$filter', '$templateCache', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
 
-    constructor($scope, $state, $templateCache, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
+    constructor($scope, $state, $filter, $templateCache, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
         const $ctrl = this;
 
         const companySelectOptions = [];
         const countrySelectOptions = [];
 
+        const dtFilter = $filter('date');
+
         $ctrl.params = {
             startDate: new Date()
         };
@@ -82,7 +93,7 @@ export default class IgniteListOfRegisteredUsersCtrl {
         };
 
         const showActivities = (user) => {
-            return new ActivitiesUserDialog({ user, params: $ctrl.params });
+            return new ActivitiesUserDialog({ user });
         };
 
         $ctrl.gridOptions = {
@@ -91,14 +102,17 @@ export default class IgniteListOfRegisteredUsersCtrl {
             columnDefs,
             categories,
             headerTemplate: $templateCache.get(headerTemplate),
+            rowTemplate,
             enableFiltering: true,
-            enableRowSelection: false,
+            enableRowSelection: true,
             enableRowHeaderSelection: false,
             enableColumnMenus: false,
             multiSelect: false,
             modifierKeysToMultiSelect: true,
             noUnselect: true,
             fastWatch: true,
+            exporterSuppressColumns: ['actions'],
+            exporterCsvColumnSeparator: ';',
             onRegisterApi: (api) => {
                 $ctrl.gridApi = api;
 
@@ -139,14 +153,14 @@ export default class IgniteListOfRegisteredUsersCtrl {
                 .then((data) => $ctrl.adjustHeight(data.length));
         };
 
-        $scope.$watch(() => $ctrl.params.startDate, () => {
-            const endDate = new Date($ctrl.params.startDate);
+        $scope.$watch(() => $ctrl.params.startDate, (dt) => {
+            $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(dt, 'yyyy_MM')}.csv`;
 
-            endDate.setMonth(endDate.getMonth() + 1);
+            const endDate = new Date(dt);
 
-            $ctrl.params.endDate = endDate;
+            endDate.setMonth(endDate.getMonth() + 1);
 
-            reloadUsers($ctrl.params);
+            reloadUsers({startDate: dtFilter(dt, 'yyyy-MM-dd'), endDate: dtFilter(endDate, 'yyyy-MM-dd')});
         });
     }
 
diff --git a/modules/web-console/frontend/app/core/activities/Activities.data.js b/modules/web-console/frontend/app/core/activities/Activities.data.js
index 8a67a97a5dd43..8d9447cd2572d 100644
--- a/modules/web-console/frontend/app/core/activities/Activities.data.js
+++ b/modules/web-console/frontend/app/core/activities/Activities.data.js
@@ -31,9 +31,4 @@ export default class ActivitiesData {
 
         return this.$http.post('/api/v1/activities/page', { group, action });
     }
-
-    listByUser(userId, params) {
-        return this.$http.get(`/api/v1/activities/user/${userId}`, { params })
-            .then(({ data }) => data);
-    }
 }
diff --git a/modules/web-console/frontend/app/data/i18n.js b/modules/web-console/frontend/app/data/i18n.js
index bc8c700d8bfa6..3385f6061c911 100644
--- a/modules/web-console/frontend/app/data/i18n.js
+++ b/modules/web-console/frontend/app/data/i18n.js
@@ -23,6 +23,7 @@ export default {
     '/configuration/domains': 'Configure domain model',
     '/configuration/igfs': 'Configure IGFS',
     '/configuration/summary': 'Configurations summary',
+    '/configuration/download': 'Download project',
     '/demo/resume': 'Demo resume',
     '/demo/reset': 'Demo reset',
     '/queries/execute': 'Query execute',
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
index 477cf20999271..0ae12695dd5f6 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-export default ['$scope', 'IgniteMavenGenerator', 'IgniteVersion', function($scope, maven, Version) {
+export default ['$scope', 'IgniteMavenGenerator', function($scope, maven) {
     const ctrl = this;
 
     // Watchers definition.
@@ -25,7 +25,7 @@ export default ['$scope', 'IgniteMavenGenerator', 'IgniteVersion', function($sco
         if (!value)
             return;
 
-        ctrl.data = maven.generate($scope.cluster, Version.productVersion().ignite).asString();
+        ctrl.data = maven.generate($scope.cluster);
     };
 
     // Setup watchers.
diff --git a/modules/web-console/frontend/app/modules/agent/agent.module.js b/modules/web-console/frontend/app/modules/agent/agent.module.js
index d6fc86311cf24..7ac39d10f3594 100644
--- a/modules/web-console/frontend/app/modules/agent/agent.module.js
+++ b/modules/web-console/frontend/app/modules/agent/agent.module.js
@@ -62,21 +62,6 @@ class IgniteAgentMonitor {
                 this._scope.$$postDigest(() => $state.go(this._scope.backState));
         };
 
-        this._scope.downloadAgent = () => {
-            const lnk = document.createElement('a');
-
-            lnk.setAttribute('href', '/api/v1/agent/download/zip');
-            lnk.setAttribute('target', '_self');
-            lnk.setAttribute('download', null);
-            lnk.style.display = 'none';
-
-            document.body.appendChild(lnk);
-
-            lnk.click();
-
-            document.body.removeChild(lnk);
-        };
-
         this._scope.hasAgents = null;
         this._scope.showModal = false;
 
diff --git a/modules/web-console/frontend/app/modules/configuration/Version.service.js b/modules/web-console/frontend/app/modules/configuration/Version.service.js
index f0e9c4c2a2a89..746b1edd9b090 100644
--- a/modules/web-console/frontend/app/modules/configuration/Version.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/Version.service.js
@@ -23,6 +23,9 @@ const VERSION_MATCHER = /(\d+)\.(\d+)\.(\d+)([-.]([^0123456789][^-]+)(-SNAPSHOT)
 const numberComparator = (a, b) => a > b ? 1 : a < b ? -1 : 0;
 
 export default class IgniteVersion {
+    /** Current product version. */
+    static ignite = '1.8.0';
+
     /**
      * Tries to parse product version from it's string representation.
      *
@@ -73,16 +76,6 @@ export default class IgniteVersion {
         return numberComparator(pa.revTs, pb.revTs);
     }
 
-    /**
-     * Return current product version.
-     * @returns {{ignite: string}}
-     */
-    productVersion() {
-        return {
-            ignite: '1.8.0'
-        };
-    }
-
     /**
      * Check if node version is newer or same
      * @param {String} nodeVer
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js
index 2e017610e0e81..23a9c4e811656 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js
@@ -16,6 +16,7 @@
  */
 
 import StringBuilder from './StringBuilder';
+import IgniteVersion from 'app/modules/configuration/Version.service';
 
 // Java built-in class names.
 import POM_DEPENDENCIES from 'app/data/pom-dependencies.json';
@@ -142,11 +143,10 @@ export default class IgniteMavenGenerator {
      * Generate pom.xml.
      *
      * @param cluster Cluster  to take info about dependencies.
-     * @param version Ignite version for Ignite dependencies.
-     * @param sb Resulting output with generated pom.
+     * @param version Version for Ignite dependencies.
      * @returns {string} Generated content.
      */
-    generate(cluster, version, sb = new StringBuilder()) {
+    generate(cluster, version = IgniteVersion.ignite) {
         const caches = cluster.caches;
         const deps = [];
         const storeDeps = [];
@@ -162,6 +162,8 @@ export default class IgniteMavenGenerator {
                 this.addDependency(deps, 'org.apache.ignite', 'ignite-extdata-p2p', version);
         });
 
+        const sb = new StringBuilder();
+
         sb.append('<?xml version="1.0" encoding="UTF-8"?>');
 
         sb.emptyLine();
@@ -229,6 +231,6 @@ export default class IgniteMavenGenerator {
 
         this.build(sb, cluster, excludeGroupIds);
 
-        return sb;
+        return sb.asString();
     }
 }
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
index 6b240017ab017..070b6ce87aa69 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
@@ -17,8 +17,6 @@
 
 import JSZip from 'jszip';
 
-import IgniteVersion from 'app/modules/configuration/Version.service';
-
 import MavenGenerator from 'app/modules/configuration/generator/Maven.service';
 import DockerGenerator from 'app/modules/configuration/generator/Docker.service';
 import ReadmeGenerator from 'app/modules/configuration/generator/Readme.service';
@@ -28,8 +26,6 @@ import ConfigurationGenerator from 'app/modules/configuration/generator/Configur
 import JavaTransformer from 'app/modules/configuration/generator/JavaTransformer.service';
 import SpringTransformer from 'app/modules/configuration/generator/SpringTransformer.service';
 
-const Version = new IgniteVersion();
-
 const maven = new MavenGenerator();
 const docker = new DockerGenerator();
 const readme = new ReadmeGenerator();
@@ -100,7 +96,7 @@ onmessage = function(e) {
     zip.file(`${startupPath}/ClientNodeCodeStartup.java`, java.nodeStartup(cluster, 'startup.ClientNodeCodeStartup',
         'ClientConfigurationFactory.createConfiguration()', 'config.ClientConfigurationFactory', clientNearCaches));
 
-    zip.file('pom.xml', maven.generate(cluster, Version.productVersion().ignite).asString());
+    zip.file('pom.xml', maven.generate(cluster));
 
     zip.file('README.txt', readme.generate());
     zip.file('jdbc-drivers/README.txt', readme.generateJDBC());
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 651f496fa78c7..2d126550f16b4 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -29,100 +29,100 @@
     "win32"
   ],
   "dependencies": {
-    "angular": "~1.5.9",
-    "angular-acl": "~0.1.7",
-    "angular-animate": "~1.5.9",
-    "angular-aria": "~1.5.9",
-    "angular-cookies": "~1.5.9",
-    "angular-drag-and-drop-lists": "~1.4.0",
-    "angular-gridster": "~0.13.3",
-    "angular-motion": "~0.4.4",
-    "angular-nvd3": "~1.0.9",
-    "angular-retina": "~0.3.13",
-    "angular-sanitize": "~1.5.9",
-    "angular-smart-table": "~2.1.8",
-    "angular-socket-io": "~0.7.0",
-    "angular-strap": "~2.3.8",
-    "angular-touch": "~1.5.9",
-    "angular-translate": "~2.13.1",
-    "angular-tree-control": "~0.2.26",
-    "angular-ui-grid": "~4.0.0",
-    "angular-ui-router": "~0.3.1",
-    "bootstrap-sass": "~3.3.6",
-    "brace": "~0.8.0",
-    "es6-promise": "~3.3.1",
-    "file-saver": "~1.3.2",
-    "font-awesome": "~4.7.0",
-    "glob": "~7.1.1",
-    "jquery": "~3.1.1",
-    "jszip": "~3.1.3",
-    "lodash": "~4.17.2",
+    "angular": "1.5.11",
+    "angular-acl": "0.1.7",
+    "angular-animate": "1.5.11",
+    "angular-aria": "1.5.11",
+    "angular-cookies": "1.5.11",
+    "angular-drag-and-drop-lists": "1.4.0",
+    "angular-gridster": "0.13.14",
+    "angular-motion": "0.4.4",
+    "angular-nvd3": "1.0.9",
+    "angular-retina": "0.4.0",
+    "angular-sanitize": "1.5.11",
+    "angular-smart-table": "2.1.8",
+    "angular-socket-io": "0.7.0",
+    "angular-strap": "2.3.12",
+    "angular-touch": "1.5.11",
+    "angular-translate": "2.14.0",
+    "angular-tree-control": "0.2.28",
+    "angular-ui-grid": "4.0.2",
+    "angular-ui-router": "0.4.2",
+    "bootstrap-sass": "3.3.7",
+    "brace": "0.8.0",
+    "es6-promise": "3.3.1",
+    "file-saver": "1.3.3",
+    "font-awesome": "4.7.0",
+    "glob": "7.1.1",
+    "jquery": "3.1.1",
+    "jszip": "3.1.3",
+    "lodash": "4.17.4",
     "nvd3": "1.8.4",
-    "raleway-webfont": "~3.0.1",
-    "roboto-font": "~0.1.0",
-    "socket.io-client": "~1.7.2",
-    "ui-router-metatags": "~1.0.3"
+    "raleway-webfont": "3.0.1",
+    "roboto-font": "0.1.0",
+    "socket.io-client": "1.7.2",
+    "ui-router-metatags": "1.0.3"
   },
   "devDependencies": {
-    "assets-webpack-plugin": "~3.5.0",
-    "autoprefixer-core": "~6.0.1",
-    "babel-core": "~6.20.0",
-    "babel-eslint": "~7.0.0",
-    "babel-loader": "~6.2.4",
-    "babel-plugin-add-module-exports": "~0.2.1",
-    "babel-plugin-transform-builtin-extend": "~1.1.0",
-    "babel-plugin-transform-runtime": "~6.15.0",
-    "babel-polyfill": "~6.20.0",
-    "babel-preset-angular": "~6.0.15",
-    "babel-preset-es2015": "~6.18.0",
-    "babel-runtime": "~6.20.0",
-    "chai": "~3.5.0",
-    "cross-env": "~1.0.7",
-    "css-loader": "~0.23.0",
-    "eslint": "~3.12.2",
-    "eslint-friendly-formatter": "~2.0.5",
-    "eslint-loader": "~1.6.1",
-    "expose-loader": "~0.7.1",
-    "extract-text-webpack-plugin": "~1.0.1",
-    "file-loader": "~0.9.0",
-    "gulp": "~3.9.1",
-    "gulp-eslint": "~3.0.0",
-    "gulp-inject": "~4.1.0",
-    "gulp-jade": "~1.1.0",
-    "gulp-ll": "~1.0.4",
-    "gulp-rimraf": "~0.2.0",
-    "gulp-sequence": "~0.4.1",
-    "gulp-util": "~3.0.7",
-    "html-loader": "~0.4.3",
-    "html-webpack-plugin": "~2.24.1",
-    "jade": "~1.11.0",
+    "assets-webpack-plugin": "3.5.1",
+    "autoprefixer-core": "6.0.1",
+    "babel-core": "6.20.0",
+    "babel-eslint": "7.0.0",
+    "babel-loader": "6.2.10",
+    "babel-plugin-add-module-exports": "0.2.1",
+    "babel-plugin-transform-builtin-extend": "1.1.2",
+    "babel-plugin-transform-runtime": "6.15.0",
+    "babel-polyfill": "6.20.0",
+    "babel-preset-angular": "6.0.15",
+    "babel-preset-es2015": "6.18.0",
+    "babel-runtime": "6.20.0",
+    "chai": "3.5.0",
+    "cross-env": "1.0.8",
+    "css-loader": "0.26.1",
+    "eslint": "3.12.2",
+    "eslint-friendly-formatter": "2.0.7",
+    "eslint-loader": "1.6.1",
+    "expose-loader": "0.7.1",
+    "extract-text-webpack-plugin": "1.0.1",
+    "file-loader": "0.9.0",
+    "gulp": "3.9.1",
+    "gulp-eslint": "3.0.1",
+    "gulp-inject": "4.1.0",
+    "gulp-jade": "1.1.0",
+    "gulp-ll": "1.0.4",
+    "gulp-rimraf": "0.2.1",
+    "gulp-sequence": "0.4.6",
+    "gulp-util": "3.0.8",
+    "html-loader": "0.4.4",
+    "html-webpack-plugin": "2.24.1",
+    "jade": "1.11.0",
     "jade-html-loader": "git://github.com/courcelan/jade-html-loader",
-    "jasmine-core": "~2.5.2",
-    "json-loader": "~0.5.4",
-    "karma": "~0.13.22",
-    "karma-babel-preprocessor": "~6.0.1",
-    "karma-jasmine": "~1.1.0",
-    "karma-mocha": "~1.3.0",
-    "karma-mocha-reporter": "~2.2.0",
-    "karma-phantomjs-launcher": "~1.0.0",
-    "karma-teamcity-reporter": "~1.0.0",
-    "karma-webpack": "~1.8.0",
-    "mocha": "~2.5.3",
-    "mocha-teamcity-reporter": "~1.1.1",
-    "morgan": "~1.7.0",
-    "ngtemplate-loader": "~1.3.1",
-    "node-sass": "~3.13.1",
-    "phantomjs-prebuilt": "~2.1.7",
-    "postcss-loader": "~0.9.1",
-    "progress-bar-webpack-plugin": "~1.9.0",
-    "require-dir": "~0.3.0",
-    "resolve-url-loader": "~1.6.1",
-    "sass-loader": "~3.1.1",
-    "style-loader": "~0.13.1",
-    "url": "~0.11.0",
-    "url-loader": "~0.5.6",
-    "webpack": "~1.14.0",
-    "webpack-dev-server": "~1.16.2",
-    "worker-loader": "~0.7.1"
+    "jasmine-core": "2.5.2",
+    "json-loader": "0.5.4",
+    "karma": "0.13.22",
+    "karma-babel-preprocessor": "6.0.1",
+    "karma-jasmine": "1.1.0",
+    "karma-mocha": "1.3.0",
+    "karma-mocha-reporter": "2.2.2",
+    "karma-phantomjs-launcher": "1.0.2",
+    "karma-teamcity-reporter": "1.0.0",
+    "karma-webpack": "1.8.1",
+    "mocha": "2.5.3",
+    "mocha-teamcity-reporter": "1.1.1",
+    "morgan": "1.7.0",
+    "ngtemplate-loader": "1.3.1",
+    "node-sass": "3.13.1",
+    "phantomjs-prebuilt": "2.1.14",
+    "postcss-loader": "0.9.1",
+    "progress-bar-webpack-plugin": "1.9.3",
+    "require-dir": "0.3.1",
+    "resolve-url-loader": "1.6.1",
+    "sass-loader": "3.2.2",
+    "style-loader": "0.13.1",
+    "url": "0.11.0",
+    "url-loader": "0.5.7",
+    "webpack": "1.14.0",
+    "webpack-dev-server": "1.16.3",
+    "worker-loader": "0.7.1"
   }
 }
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index 67cfed176d39e..ab7e3dd94286f 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -2331,14 +2331,6 @@ html,body,.splash-screen {
     .ui-grid-cell-contents > i {
         line-height: $line-height-base;
     }
-
-    .ui-grid-row:nth-child(odd):hover .ui-grid-cell {
-        background: $ignite-row-hover;
-    }
-    
-    .ui-grid-row:nth-child(even):hover .ui-grid-cell {
-        background: $ignite-row-hover;
-    }
 }
 
 .datepicker.dropdown-menu {
diff --git a/modules/web-console/frontend/public/stylesheets/variables.scss b/modules/web-console/frontend/public/stylesheets/variables.scss
index e30bbddf9a6e4..8500eace44e41 100644
--- a/modules/web-console/frontend/public/stylesheets/variables.scss
+++ b/modules/web-console/frontend/public/stylesheets/variables.scss
@@ -26,4 +26,3 @@ $ignite-border-bottom-color: $brand-primary;
 $ignite-background-color: #fff;
 $ignite-header-color: #555;
 $ignite-invalid-color: $brand-primary;
-$ignite-row-hover: #c9dde1;
diff --git a/modules/web-console/frontend/views/templates/agent-download.jade b/modules/web-console/frontend/views/templates/agent-download.jade
index f57bf1d87e21c..f5a6ba06d35e7 100644
--- a/modules/web-console/frontend/views/templates/agent-download.jade
+++ b/modules/web-console/frontend/views/templates/agent-download.jade
@@ -23,10 +23,10 @@
                     span(ng-if='!hasAgents') Connection to Ignite Web Agent is not established
                     span(ng-if='hasAgents') Connection to Ignite Node is not established
             .agent-download(ng-if='!hasAgents')
-                p Please download and run #[a(href='javascript:void(0)' ng-click='downloadAgent()') ignite-web-agent] in order to {{::agentGoal}}
+                p Please download and run #[a(href='/api/v1/agent/download/zip' target='_self') ignite-web-agent] in order to {{::agentGoal}}
                 p For run:
                 ul
-                    li Download and unzip #[a(href='javascript:void(0)' ng-click='downloadAgent()') ignite-web-agent] archive
+                    li Download and unzip #[a(href='/api/v1/agent/download/zip' target='_self') ignite-web-agent] archive
                     li Run shell file #[b ignite-web-agent.{sh|bat}]
                 p Refer to #[b README.txt] in agent folder for more information
                 .modal-advanced-options
@@ -47,4 +47,4 @@
                     li Refer to #[b README.txt] in agent folder for more information
             .modal-footer
                 button.btn.btn-default(ng-click='back()') {{::backText}}
-                button.btn.btn-primary(ng-if='!hasAgents' ng-click='downloadAgent()') Download agent
+                a.btn.btn-primary(ng-if='!hasAgents' href='/api/v1/agent/download/zip' target='_self') Download agent

From db9252c9588c671279664484bb8c397312d265e6 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 15 Feb 2017 16:08:57 +0700
Subject: [PATCH 019/516] IGNITE-4678 Node version in range.

(cherry picked from commit 2cfd55d)
---
 .../modules/configuration/Version.service.js  | 30 ++++++++++++++++---
 .../frontend/test/unit/Version.test.js        | 26 +++++++++++++++-
 .../demo/service/DemoCachesLoadService.java   |  2 +-
 .../service/DemoRandomCacheLoadService.java   |  2 +-
 4 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/configuration/Version.service.js b/modules/web-console/frontend/app/modules/configuration/Version.service.js
index 746b1edd9b090..3fc719946729f 100644
--- a/modules/web-console/frontend/app/modules/configuration/Version.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/Version.service.js
@@ -76,13 +76,35 @@ export default class IgniteVersion {
         return numberComparator(pa.revTs, pb.revTs);
     }
 
+    /**
+     * Check if node version in range
+     * @param {String} nodeVer Node version.
+     * @param {Array.<String>} ranges Version ranges to compare with.
+     * @returns {Boolean} `True` if node version is equal or greater than specified range.
+     */
+    includes(nodeVer, ...ranges) {
+        return !!_.find(ranges, ([after, before]) =>
+            this.compare(nodeVer, after) >= 0 && (_.isNil(before) || this.compare(nodeVer, before) < 0)
+        );
+    }
+
     /**
      * Check if node version is newer or same
-     * @param {String} nodeVer
-     * @param {String} sinceVer
-     * @returns {Boolean}
+     * @param {String} nodeVer Node version.
+     * @param {String} sinceVer Version to compare with.
+     * @returns {Boolean} `True` if node version is equal or greater than specified version.
      */
     since(nodeVer, sinceVer) {
-        return this.compare(nodeVer, sinceVer) >= 0;
+        return this.includes(nodeVer, [sinceVer]);
+    }
+
+    /**
+     * Check whether node version before than specified version.
+     * @param {String} nodeVer Node version.
+     * @param {String} sinceVer Version to compare with.
+     * @return {Boolean} `True` if node version before than specified version.
+     */
+    before(nodeVer, sinceVer) {
+        return !this.since(nodeVer, sinceVer);
     }
 }
diff --git a/modules/web-console/frontend/test/unit/Version.test.js b/modules/web-console/frontend/test/unit/Version.test.js
index 2d75ab583a2c6..72685eae03239 100644
--- a/modules/web-console/frontend/test/unit/Version.test.js
+++ b/modules/web-console/frontend/test/unit/Version.test.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import VersionService from '../../app/modules/configuration/Version.service.js';
+import VersionService from '../../app/modules/configuration/Version.service';
 
 const INSTANCE = new VersionService();
 
@@ -53,6 +53,7 @@ suite('VersionServiceTestsSuite', () => {
     });
 
     test('Check since call', () => {
+        assert.equal(INSTANCE.since('1.5.0', '1.5.0'), true);
         assert.equal(INSTANCE.since('1.6.0', '1.5.0'), true);
     });
 
@@ -60,6 +61,29 @@ suite('VersionServiceTestsSuite', () => {
         assert.equal(INSTANCE.since('1.3.0', '1.5.0'), false);
     });
 
+    test('Check before call', () => {
+        assert.equal(INSTANCE.before('1.5.0', '1.5.0'), false);
+        assert.equal(INSTANCE.before('1.5.0', '1.6.0'), true);
+    });
+
+    test('Check wrong before call', () => {
+        assert.equal(INSTANCE.before('1.5.0', '1.3.0'), false);
+    });
+
+    test('Check includes call', () => {
+        assert.equal(INSTANCE.includes('1.5.4', ['1.5.5', '1.6.0'], ['1.6.2']), false);
+        assert.equal(INSTANCE.includes('1.5.5', ['1.5.5', '1.6.0'], ['1.6.2']), true);
+        assert.equal(INSTANCE.includes('1.5.11', ['1.5.5', '1.6.0'], ['1.6.2']), true);
+        assert.equal(INSTANCE.includes('1.6.0', ['1.5.5', '1.6.0'], ['1.6.2']), false);
+        assert.equal(INSTANCE.includes('1.6.1', ['1.5.5', '1.6.0'], ['1.6.2']), false);
+        assert.equal(INSTANCE.includes('1.6.2', ['1.5.5', '1.6.0'], ['1.6.2']), true);
+        assert.equal(INSTANCE.includes('1.6.3', ['1.5.5', '1.6.0'], ['1.6.2']), true);
+    });
+
+    test('Check wrong before call', () => {
+        assert.equal(INSTANCE.before('1.5.0', '1.3.0'), false);
+    });
+
     test('Parse 1.7.0-SNAPSHOT', () => {
         const version = INSTANCE.parse('1.7.0-SNAPSHOT');
         assert.equal(version.major, 1);
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
index 911764604502f..fbfa2ae6e09e2 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
@@ -114,7 +114,7 @@ public DemoCachesLoadService(int cnt) {
     /** {@inheritDoc} */
     @Override public void cancel(ServiceContext ctx) {
         if (cachePool != null)
-            cachePool.shutdown();
+            cachePool.shutdownNow();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
index 57b26a2102ded..c704dbe09cf06 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java
@@ -62,7 +62,7 @@ public DemoRandomCacheLoadService(int cnt) {
     /** {@inheritDoc} */
     @Override public void cancel(ServiceContext ctx) {
         if (cachePool != null)
-            cachePool.shutdown();
+            cachePool.shutdownNow();
     }
 
     /** {@inheritDoc} */

From 58c0c49d31605bf4608e7ee97099b75b324a782f Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 16 Feb 2017 10:41:30 +0700
Subject: [PATCH 020/516] IGNITE-4472 Fixed became this user.

(cherry picked from commit ee832e4)
---
 modules/web-console/backend/routes/profile.js              | 4 ++--
 modules/web-console/backend/services/sessions.js           | 6 +-----
 modules/web-console/backend/services/users.js              | 7 ++-----
 .../frontend/views/templates/agent-download.jade           | 4 ++--
 4 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/modules/web-console/backend/routes/profile.js b/modules/web-console/backend/routes/profile.js
index 1d6fccb8213f3..76edf721fde56 100644
--- a/modules/web-console/backend/routes/profile.js
+++ b/modules/web-console/backend/routes/profile.js
@@ -50,7 +50,7 @@ module.exports.factory = function(_, express, mongo, usersService) {
                     if (becomeUsed) {
                         req.session.viewedUser = user;
 
-                        return user;
+                        return req.user;
                     }
 
                     return new Promise((resolve, reject) => {
@@ -64,7 +64,7 @@ module.exports.factory = function(_, express, mongo, usersService) {
                         });
                     });
                 })
-                .then(() => usersService.get(req.user, req.session.viewedUser))
+                .then((user) => usersService.get(user, req.session.viewedUser))
                 .then(res.api.ok)
                 .catch(res.api.error);
         });
diff --git a/modules/web-console/backend/services/sessions.js b/modules/web-console/backend/services/sessions.js
index 7f62a606c9369..7f3fc7351b19b 100644
--- a/modules/web-console/backend/services/sessions.js
+++ b/modules/web-console/backend/services/sessions.js
@@ -42,11 +42,7 @@ module.exports.factory = (_, mongo, errors) => {
                 return Promise.reject(new errors.IllegalAccessError('Became this user is not permitted. Only administrators can perform this actions.'));
 
             return mongo.Account.findById(viewedUserId).lean().exec()
-                .then((viewedUser) => {
-                    viewedUser.token = session.req.user.token;
-
-                    session.viewedUser = viewedUser;
-                });
+                .then((viewedUser) => session.viewedUser = viewedUser);
         }
 
         /**
diff --git a/modules/web-console/backend/services/users.js b/modules/web-console/backend/services/users.js
index 2dd603f193382..0aff45f15356b 100644
--- a/modules/web-console/backend/services/users.js
+++ b/modules/web-console/backend/services/users.js
@@ -212,11 +212,8 @@ module.exports.factory = (_, mongo, settings, spacesService, mailsService, activ
 
             const becomeUsed = viewedUser && user.admin;
 
-            if (becomeUsed) {
-                user = viewedUser;
-
-                user.becomeUsed = true;
-            }
+            if (becomeUsed)
+                user = _.extend({}, viewedUser, {becomeUsed: true, becameToken: user.token});
             else
                 user = user.toJSON();
 
diff --git a/modules/web-console/frontend/views/templates/agent-download.jade b/modules/web-console/frontend/views/templates/agent-download.jade
index f5a6ba06d35e7..b913636cb0fcd 100644
--- a/modules/web-console/frontend/views/templates/agent-download.jade
+++ b/modules/web-console/frontend/views/templates/agent-download.jade
@@ -34,8 +34,8 @@
                     i.fa.fa-chevron-circle-right(ng-show='!agentLoad.showToken' ng-click='agentLoad.showToken = ! agentLoad.showToken')
                     a(ng-click='agentLoad.showToken = ! agentLoad.showToken') {{agentLoad.showToken ? 'Hide security token...' : 'Show security token...'}}
                 .details-row(ng-show='agentLoad.showToken')
-                    label.labelField Security token: {{user.token}}
-                    i.tipLabel.fa.fa-clipboard(ignite-copy-to-clipboard='{{user.token}}' bs-tooltip='' data-title='Copy security token to clipboard')
+                    label.labelField Security token: {{user.becameToken || user.token}}
+                    i.tipLabel.fa.fa-clipboard(ignite-copy-to-clipboard='{{user.becameToken || user.token}}' bs-tooltip='' data-title='Copy security token to clipboard')
                     i.tipLabel.icon-help(ng-if=lines bs-tooltip='' data-title='The security token is used for authorization of web agent')
             .agent-download(ng-if='hasAgents')
                 p Connection to Ignite Web Agent is established, but agent failed to connect to Ignite Node

From 2ccbf32ea3ecaff1068832accf37235a32734b4b Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 16 Feb 2017 14:22:22 +0700
Subject: [PATCH 021/516] IGNITE-4472 Minor UI fix.

(cherry picked from commit 97c7ed7)
---
 .../list-of-registered-users.column-defs.js   |  8 ++--
 .../list-of-registered-users.controller.js    | 18 +++++++++
 .../list-of-registered-users.jade             | 16 +++++---
 .../ui-grid-settings/ui-grid-settings.scss    | 39 ++++++++++++++++---
 4 files changed, 66 insertions(+), 15 deletions(-)

diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
index 4dc465505b6ab..e6ba842aa055b 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
@@ -49,7 +49,7 @@ const ACTIONS_TEMPLATE = `
 const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
 
 export default [
-    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, pinnedLeft: true},
+    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 65, width: 65, enableFiltering: false, enableSorting: false, pinnedLeft: true},
     {displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by name...' }, pinnedLeft: true},
     {displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
     {displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 160, enableFiltering: true},
@@ -64,9 +64,9 @@ export default [
     // Activities Total
     {displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Total count of configuration usages', minWidth: 50, width: 50, enableFiltering: false},
     {displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Total count of queries usages', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Total count of demo startup', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Total count of agent downloads', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Str', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Total count of agent startup', minWidth: 50, width: 50, enableFiltering: false},
+    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Total count of demo startup', minWidth: 60, width: 60, enableFiltering: false},
+    {displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Total count of agent downloads', minWidth: 55, width: 55, enableFiltering: false},
+    {displayName: 'Starts', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Total count of agent startup', minWidth: 60, width: 60, enableFiltering: false},
     // Activities Configuration
     {displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration clusters', minWidth: 50, width: 80, enableFiltering: false, visible: false},
     {displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration model', minWidth: 50, width: 80, enableFiltering: false, visible: false},
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 1f2a348daad7a..5761073509d7e 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -96,6 +96,18 @@ export default class IgniteListOfRegisteredUsersCtrl {
             return new ActivitiesUserDialog({ user });
         };
 
+        const companiesExcludeFilter = (renderableRows) => {
+            if (_.isNil($ctrl.params.companiesExclude))
+                return renderableRows;
+
+            _.forEach(renderableRows, (row) => {
+                row.visible = _.isEmpty($ctrl.params.companiesExclude) ||
+                    row.entity.company.toLowerCase().indexOf($ctrl.params.companiesExclude.toLowerCase()) === -1;
+            });
+
+            return renderableRows;
+        };
+
         $ctrl.gridOptions = {
             data: [],
             columnVirtualizationThreshold: 30,
@@ -120,6 +132,8 @@ export default class IgniteListOfRegisteredUsersCtrl {
                 api.removeUser = removeUser;
                 api.toggleAdmin = toggleAdmin;
                 api.showActivities = showActivities;
+
+                api.grid.registerRowsProcessor(companiesExcludeFilter, 300);
             }
         };
 
@@ -153,6 +167,10 @@ export default class IgniteListOfRegisteredUsersCtrl {
                 .then((data) => $ctrl.adjustHeight(data.length));
         };
 
+        $scope.$watch(() => $ctrl.params.companiesExclude, () => {
+            $ctrl.gridApi.grid.refreshRows();
+        });
+
         $scope.$watch(() => $ctrl.params.startDate, (dt) => {
             $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(dt, 'yyyy_MM')}.csv`;
 
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
index efed9c0edee16..119591036d9da 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
@@ -39,16 +39,20 @@ mixin grid-settings()
         +grid-settings
         label Total users: 
             strong {{ $ctrl.gridOptions.data.length }}&nbsp;&nbsp;&nbsp;
-        label Showing users: 
+        label Showing users:
             strong {{ $ctrl.gridApi.grid.getVisibleRows().length }}
             sub(ng-show='users.length === $ctrl.gridApi.grid.getVisibleRows().length') all
-        div.ui-grid-settings-dateperiod
-            form(ng-form=form novalidate)
-                -var form = 'admin'
 
+        form.pull-right(ng-form=form novalidate)
+            -var form = 'admin'
+
+            button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
+
+            .ui-grid-settings-dateperiod
                 +ignite-form-field-datepicker('Period:', '$ctrl.params.startDate', '"period"')
-                
-                button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
+
+            .ui-grid-settings-filter
+                +ignite-form-field-text('Exclude:', '$ctrl.params.companiesExclude', '"exclude"', false, false, 'Exclude by company name...')
 
     .panel-collapse
         .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-exporter ui-grid-pinning)
diff --git a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
index 3016488f6bb6e..bc16271a57bc0 100644
--- a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
@@ -36,6 +36,40 @@
         cursor: pointer;
     }
 
+    .btn {
+        float: right;
+
+        line-height: 20px;
+        margin-right: 0;
+    }
+
+    &-filter {
+        float: right;
+
+        .ignite-form-field {
+            width: 260px;
+            margin-right: 10px;
+
+            &__label {
+            }
+
+            &__control {
+            }
+
+            &:nth-child(1) {
+                float: left;
+
+                .ignite-form-field__label {
+                    width: 30%;
+                }
+
+                .ignite-form-field__control {
+                    width: 70%;
+                }
+            }
+        }
+    }
+
     &-dateperiod {
         float: right;
 
@@ -61,10 +95,5 @@
                 }
             }
         }
-
-        .btn {
-            line-height: 20px;
-            margin-right: 0;
-        }
     }
 }

From ef4886d5be7c708b917e97b1cd5fd766b2ad8450 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 16 Feb 2017 18:38:40 +0700
Subject: [PATCH 022/516] IGNITE-4472 Minor UI fix.

(cherry picked from commit d4efbf3)
---
 .../list-of-registered-users.controller.js                      | 2 +-
 .../app/components/ui-grid-settings/ui-grid-settings.scss       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 5761073509d7e..272681a5f576b 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -133,7 +133,7 @@ export default class IgniteListOfRegisteredUsersCtrl {
                 api.toggleAdmin = toggleAdmin;
                 api.showActivities = showActivities;
 
-                api.grid.registerRowsProcessor(companiesExcludeFilter, 300);
+                api.grid.registerRowsProcessor(companiesExcludeFilter, 50);
             }
         };
 
diff --git a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
index bc16271a57bc0..4beb2a1baa40a 100644
--- a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
@@ -47,7 +47,7 @@
         float: right;
 
         .ignite-form-field {
-            width: 260px;
+            width: 280px;
             margin-right: 10px;
 
             &__label {

From 05788b3188b30b5a3b39a75fe66301e03658408f Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 17 Feb 2017 12:14:53 +0300
Subject: [PATCH 023/516] IGNITE-3429: Added BinaryResolver configuration
 samples for  org.hibernate.cache.spi.CacheKey. This closes #1516.

---
 .../Hibernate5CacheKeyTypeConfiguration.java  | 52 +++++++++++++++++++
 .../HibernateCacheKeyTypeConfiguration.java   | 51 ++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java

diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
new file mode 100644
index 0000000000000..886f69b2500ba
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate.config;
+
+import java.util.Objects;
+import org.apache.ignite.binary.BinaryAbstractIdentityResolver;
+import org.apache.ignite.binary.BinaryIdentityResolver;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryTypeConfiguration;
+
+/**
+ * This configuration provides correct {@link BinaryIdentityResolver} implementation
+ * for Hibernate CacheKey class can be used as a key object.
+ *
+ * Note: for Hibernate version < 5.0 {@link HibernateCacheKeyTypeConfiguration} should be used.
+
+ */
+public class Hibernate5CacheKeyTypeConfiguration extends BinaryTypeConfiguration {
+
+    /** {@inheritDoc} */
+    public Hibernate5CacheKeyTypeConfiguration() {
+        super("org.hibernate.cache.internal.CacheKeyImplementation");
+
+        setIdentityResolver(new BinaryAbstractIdentityResolver() {
+            @Override protected int hashCode0(BinaryObject obj) {
+                return obj.field("id").hashCode();
+            }
+
+            @Override protected boolean equals0(BinaryObject o1, BinaryObject o2) {
+                Object obj0 = o1.field("id");
+                Object obj1 = o2.field("id");
+
+                return Objects.equals(obj0, obj1);
+            }
+        });
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java
new file mode 100644
index 0000000000000..c54292e6f40e6
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate.config;
+
+import java.util.Objects;
+import org.apache.ignite.binary.BinaryAbstractIdentityResolver;
+import org.apache.ignite.binary.BinaryIdentityResolver;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryTypeConfiguration;
+
+/**
+ * This configuration provides correct {@link BinaryIdentityResolver} implementation
+ * for Hibernate CacheKey class can be used as a key object.
+ *
+ * Note: for Hibernate version >= 5.0 {@link Hibernate5CacheKeyTypeConfiguration} should be used.
+ */
+public class HibernateCacheKeyTypeConfiguration extends BinaryTypeConfiguration {
+
+    /** {@inheritDoc} */
+    public HibernateCacheKeyTypeConfiguration() {
+        super("org.hibernate.cache.spi.CacheKey");
+
+        setIdentityResolver(new BinaryAbstractIdentityResolver() {
+            @Override protected int hashCode0(BinaryObject obj) {
+                return obj.field("key").hashCode();
+            }
+
+            @Override protected boolean equals0(BinaryObject o1, BinaryObject o2) {
+                Object obj0 = o1.field("key");
+                Object obj1 = o2.field("key");
+
+                return Objects.equals(obj0, obj1);
+            }
+        });
+    }
+}

From 1f881aa70a3894af01135f4cc5e341a8130462c2 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 17 Feb 2017 12:34:41 +0300
Subject: [PATCH 024/516] IGNITE-4147 - Throw exception on connecting node to
 cluster with different SSL configuration

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  30 ++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  19 ++
 .../TcpDiscoverySslSecuredUnsecuredTest.java  | 185 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   2 +
 4 files changed, 235 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 932e7d1e754a1..95e2cda4f6907 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -20,6 +20,7 @@
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StreamCorruptedException;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
@@ -44,6 +45,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.atomic.AtomicReference;
+import javax.net.ssl.SSLException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
@@ -587,6 +589,8 @@ else if (addrs.isEmpty()) {
 
         int connectAttempts = 1;
 
+        int sslConnectAttempts = 3;
+
         UUID locNodeId = getLocalNodeId();
 
         IgniteSpiOperationTimeoutHelper timeoutHelper = new IgniteSpiOperationTimeoutHelper(spi);
@@ -662,6 +666,22 @@ else if (addrs.isEmpty()) {
 
                 errs.add(e);
 
+                if (X.hasCause(e, SSLException.class)) {
+                    if (--sslConnectAttempts == 0)
+                        throw new IgniteSpiException("Unable to establish secure connection. " +
+                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
+                    continue;
+                }
+
+                if (X.hasCause(e, StreamCorruptedException.class)) {
+                    if (--sslConnectAttempts == 0)
+                        throw new IgniteSpiException("Unable to establish plain connection. " +
+                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
+                    continue;
+                }
+
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
@@ -1593,7 +1613,15 @@ private void tryJoin() throws InterruptedException {
 
             joinCnt++;
 
-            T2<SocketStream, Boolean> joinRes = joinTopology(false, spi.joinTimeout);
+            T2<SocketStream, Boolean> joinRes;
+            try {
+                joinRes = joinTopology(false, spi.joinTimeout);
+            }
+            catch (IgniteSpiException e) {
+                joinError(e);
+
+                return;
+            }
 
             if (joinRes == null) {
                 if (join)
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index d462ac2638fd2..4600be094c840 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -24,6 +24,7 @@
 import java.io.ObjectStreamException;
 import java.io.OutputStream;
 import java.io.Serializable;
+import java.io.StreamCorruptedException;
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -1109,6 +1110,8 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
         int connectAttempts = 1;
 
+        int sslConnectAttempts = 3;
+
         boolean joinReqSent;
 
         UUID locNodeId = getLocalNodeId();
@@ -1220,6 +1223,22 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
                 errs.add(e);
 
+                if (X.hasCause(e, SSLException.class)) {
+                    if (--sslConnectAttempts == 0)
+                        throw new IgniteException("Unable to establish secure connection. " +
+                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
+                    continue;
+                }
+
+                if (X.hasCause(e, StreamCorruptedException.class)) {
+                    if (--sslConnectAttempts == 0)
+                        throw new IgniteException("Unable to establish plain connection. " +
+                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+
+                    continue;
+                }
+
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java
new file mode 100644
index 0000000000000..ca34f779760cf
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslSecuredUnsecuredTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StreamCorruptedException;
+import java.net.Socket;
+import java.util.concurrent.Callable;
+import javax.net.ssl.SSLException;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Tests cases when node connects to cluster with different SSL configuration.
+ * Exception with meaningful message should be thrown.
+ */
+public class TcpDiscoverySslSecuredUnsecuredTest extends GridCommonAbstractTest {
+    /** */
+    private volatile TcpDiscoverySpi spi;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(gridName.contains("client"));
+
+        if (gridName.contains("ssl"))
+            cfg.setSslContextFactory(GridTestUtils.sslFactory());
+
+        if (spi != null) {
+            final TcpDiscoveryIpFinder finder = ((TcpDiscoverySpi)cfg.getDiscoverySpi()).getIpFinder();
+
+            spi.setIpFinder(finder);
+
+            cfg.setDiscoverySpi(spi);
+        }
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSecuredUnsecuredServerConnection() throws Exception {
+        checkConnection("plain-server", "ssl-server");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testUnsecuredSecuredServerConnection() throws Exception {
+        checkConnection("ssl-server", "plain-server");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSecuredClientUnsecuredServerConnection() throws Exception {
+        checkConnection("plain-server", "ssl-client");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testUnsecuredClientSecuredServerConnection() throws Exception {
+        checkConnection("ssl-server", "plain-client");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPlainServerNodesRestart() throws Exception {
+        checkNodesRestart("plain-server-1", "plain-server-2");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSslServerNodesRestart() throws Exception {
+        checkNodesRestart("ssl-server-1", "ssl-server-2");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPlainClientNodesRestart() throws Exception {
+        checkNodesRestart("plain-server", "plain-client");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSslClientNodesRestart() throws Exception {
+        checkNodesRestart("ssl-server", "ssl-client");
+    }
+
+    /**
+     * @param name1 First grid name.
+     * @param name2 Second grid name.
+     * @throws Exception If failed.
+     */
+    private void checkNodesRestart(String name1, String name2) throws Exception {
+        startGrid(name1);
+
+        spi = new FailDiscoverySpi(!name1.contains("ssl"));
+
+        startGrid(name2);
+    }
+
+    /**
+     * @param name1 First grid name.
+     * @param name2 Second grid name.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    private void checkConnection(final String name1, final String name2) throws Exception {
+        startGrid(name1);
+
+        GridTestUtils.assertThrows(null, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                startGrid(name2);
+
+                return null;
+            }
+        }, IgniteCheckedException.class, null);
+    }
+
+    /**
+     *
+     */
+    private class FailDiscoverySpi extends TcpDiscoverySpi {
+        /** */
+        private int cnt = 1;
+
+        /** */
+        private final boolean plain;
+
+        /**
+         * @param plain Plain conection flag.
+         */
+        private FailDiscoverySpi(final boolean plain) {
+            this.plain = plain;
+        }
+
+        /** {@inheritDoc} */
+        @Override protected <T> T readMessage(final Socket sock, @Nullable final InputStream in,
+            final long timeout) throws IOException, IgniteCheckedException {
+            if (cnt-- > 0) {
+                if (plain)
+                    throw new StreamCorruptedException("Test exception");
+                else
+                    throw new SSLException("Test SSL exception");
+            }
+
+            return super.readMessage(sock, in, timeout);
+        }
+    }
+
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 548e1a5a51f81..e6b39f740a0dd 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiStartStopSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSecuredUnsecuredTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSelfTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.jdbc.TcpDiscoveryJdbcIpFinderSelfTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinderSelfTest;
@@ -89,6 +90,7 @@ public static TestSuite suite() throws Exception {
 
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
+        suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));
 
         return suite;
     }

From 11bbec487dc174fac1acd6b50c940840305bc75a Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 17 Feb 2017 17:57:50 +0700
Subject: [PATCH 025/516] IGNITE-4436 API for collecting list of running
 queries and cancel them. (cherry picked from commit
 49237343d53ee33d44e5599cd7fe7da868ee30a1)

---
 .../cache/query/GridCacheTwoStepQuery.java    |  18 +-
 .../processors/query/GridQueryIndexing.java   |  17 +-
 .../processors/query/GridQueryProcessor.java  |  26 ++-
 .../query/GridRunningQueryInfo.java           | 132 +++++++++++
 .../internal/visor/VisorMultiNodeTask.java    |   2 +-
 .../visor/query/VisorCancelQueriesTask.java   |  72 ++++++
 .../query/VisorCollectRunningQueriesTask.java |  96 ++++++++
 .../visor/query/VisorRunningQuery.java        | 132 +++++++++++
 .../processors/query/h2/IgniteH2Indexing.java |  83 ++++++-
 .../query/h2/sql/GridSqlQuerySplitter.java    |   4 +-
 .../h2/twostep/GridReduceQueryExecutor.java   |  60 ++++-
 .../cache/CacheSqlQueryValueCopySelfTest.java | 208 +++++++++++++++++-
 .../GridCacheCrossCacheQuerySelfTest.java     |   2 +-
 .../h2/GridIndexingSpiAbstractSelfTest.java   |   7 +
 14 files changed, 821 insertions(+), 38 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
index 8dcba2f2f34dc..f53936fc374a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
@@ -45,6 +45,9 @@ public class GridCacheTwoStepQuery {
     /** */
     private boolean explain;
 
+    /** */
+    private String originalSql;
+
     /** */
     private Collection<String> spaces;
 
@@ -67,10 +70,12 @@ public class GridCacheTwoStepQuery {
     private List<Integer> extraCaches;
 
     /**
+     * @param originalSql Original query SQL.
      * @param schemas Schema names in query.
      * @param tbls Tables in query.
      */
-    public GridCacheTwoStepQuery(Set<String> schemas, Set<String> tbls) {
+    public GridCacheTwoStepQuery(String originalSql, Set<String> schemas, Set<String> tbls) {
+        this.originalSql = originalSql;
         this.schemas = schemas;
         this.tbls = tbls;
     }
@@ -195,6 +200,13 @@ public void extraCaches(List<Integer> extraCaches) {
         this.extraCaches = extraCaches;
     }
 
+    /**
+     * @return Original query SQL.
+     */
+    public String originalSql() {
+        return originalSql;
+    }
+
     /**
      * @return Spaces.
      */
@@ -223,7 +235,7 @@ public Set<String> schemas() {
     public GridCacheTwoStepQuery copy(Object[] args) {
         assert !explain;
 
-        GridCacheTwoStepQuery cp = new GridCacheTwoStepQuery(schemas, tbls);
+        GridCacheTwoStepQuery cp = new GridCacheTwoStepQuery(originalSql, schemas, tbls);
 
         cp.caches = caches;
         cp.extraCaches = extraCaches;
@@ -250,4 +262,4 @@ public Set<String> tables() {
     @Override public String toString() {
         return S.toString(GridCacheTwoStepQuery.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index 539ebc07fec79..1cebbb0a54ff6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -223,8 +223,23 @@ public void store(@Nullable String spaceName, GridQueryTypeDescriptor type, Cach
      */
     public void onDisconnected(IgniteFuture<?> reconnectFut);
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> runningQueries(long duration);
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries ID's to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries);
+
     /**
      * Cancels all executing queries.
      */
     public void cancelAllQueries();
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index c2e5717fbddf8..0a0d16614a92b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -44,7 +44,6 @@
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryField;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryType;
@@ -118,7 +117,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
     private static final int QRY_DETAIL_METRICS_EVICTION_FREQ = 3_000;
 
     /** */
-    private static Set<Class<?>> SQL_TYPES = new HashSet<>(F.<Class<?>>asList(
+    private static final Set<Class<?>> SQL_TYPES = new HashSet<>(F.<Class<?>>asList(
         Integer.class,
         Boolean.class,
         Byte.class,
@@ -912,6 +911,29 @@ public <K, V> Iterator<Cache.Entry<K, V>> queryLocal(
         }
     }
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+        if (moduleEnabled())
+            return idx.runningQueries(duration);
+
+        return Collections.emptyList();
+    }
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries ID's to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries) {
+        if (moduleEnabled())
+            idx.cancelQueries(queries);
+    }
+
     /**
      * @param sqlQry Sql query.
      * @param params Params.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
new file mode 100644
index 0000000000000..d77c8c01a8ede
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
+
+/**
+ * Query descriptor.
+ */
+public class GridRunningQueryInfo {
+    /** */
+    private final long id;
+
+    /** */
+    private final String qry;
+
+    /** Query type. */
+    private final GridCacheQueryType qryType;
+
+    /** */
+    private final String cache;
+
+    /** */
+    private final long startTime;
+
+    /** */
+    private final GridQueryCancel cancel;
+
+    /** */
+    private final boolean loc;
+
+    /**
+     * @param id Query ID.
+     * @param qry Query text.
+     * @param qryType Query type.
+     * @param cache Cache where query was executed.
+     * @param startTime Query start time.
+     * @param cancel Query cancel.
+     * @param loc Local query flag.
+     */
+    public GridRunningQueryInfo(Long id, String qry, GridCacheQueryType qryType, String cache, long startTime,
+        GridQueryCancel cancel, boolean loc) {
+        this.id = id;
+        this.qry = qry;
+        this.qryType = qryType;
+        this.cache = cache;
+        this.startTime = startTime;
+        this.cancel = cancel;
+        this.loc = loc;
+    }
+
+    /**
+     * @return Query ID.
+     */
+    public Long id() {
+        return id;
+    }
+
+    /**
+     * @return Query text.
+     */
+    public String query() {
+        return qry;
+    }
+
+    /**
+     * @return Query type.
+     */
+    public GridCacheQueryType queryType() {
+        return qryType;
+    }
+
+    /**
+     * @return Cache where query was executed.
+     */
+    public String cache() {
+        return cache;
+    }
+
+    /**
+     * @return Query start time.
+     */
+    public long startTime() {
+        return startTime;
+    }
+
+    /**
+     * @param curTime Current time.
+     * @param duration Duration of long query.
+     * @return {@code true} if this query should be considered as long running query.
+     */
+    public boolean longQuery(long curTime, long duration) {
+        return curTime - startTime > duration;
+    }
+
+    /**
+     * Cancel query.
+     */
+    public void cancel() {
+        if (cancel != null)
+            cancel.cancel();
+    }
+
+    /**
+     * @return {@code true} if query can be cancelled.
+     */
+    public boolean cancelable() {
+        return cancel != null;
+    }
+
+    /**
+     * @return {@code true} if query is local.
+     */
+    public boolean local() {
+        return loc;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
index 57f134698ea56..ece1a17953f39 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
@@ -130,4 +130,4 @@ protected Map<? extends ComputeJob, ClusterNode> map0(List<ClusterNode> subgrid,
                 logFinish(ignite.log(), getClass(), start);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
new file mode 100644
index 0000000000000..a6f2d821f15a0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task to cancel queries.
+ */
+@GridInternal
+public class VisorCancelQueriesTask extends VisorOneNodeTask<Collection<Long>, Void> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCancelQueriesJob job(Collection<Long> arg) {
+        return new VisorCancelQueriesJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected Void reduce0(List<ComputeJobResult> results) throws IgniteException {
+        return null;
+    }
+
+    /**
+     * Job to cancel queries on node.
+     */
+    private static class VisorCancelQueriesJob extends VisorJob<Collection<Long>, Void> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with specified argument.
+         *
+         * @param arg Job argument.
+         * @param debug Flag indicating whether debug information should be printed into node log.
+         */
+        protected VisorCancelQueriesJob(@Nullable Collection<Long> arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Void run(@Nullable Collection<Long> queries) throws IgniteException {
+            ignite.context().query().cancelQueries(queries);
+
+            return null;
+        }
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
new file mode 100644
index 0000000000000..2b40e61e74f50
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task to collect currently running queries.
+ */
+@GridInternal
+public class VisorCollectRunningQueriesTask extends VisorMultiNodeTask<Long, Map<UUID, Collection<VisorRunningQuery>>, Collection<VisorRunningQuery>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCollectRunningQueriesJob job(Long arg) {
+        return new VisorCollectRunningQueriesJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected Map<UUID, Collection<VisorRunningQuery>> reduce0(List<ComputeJobResult> results) throws IgniteException {
+        Map<UUID, Collection<VisorRunningQuery>> map = new HashMap<>();
+
+        for (ComputeJobResult res : results)
+            if (res.getException() == null) {
+                Collection<VisorRunningQuery> queries = res.getData();
+
+                map.put(res.getNode().id(), queries);
+            }
+
+        return map;
+    }
+
+    /**
+     * Job to collect currently running queries from node.
+     */
+    private static class VisorCollectRunningQueriesJob extends VisorJob<Long, Collection<VisorRunningQuery>> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with specified argument.
+         *
+         * @param arg Job argument.
+         * @param debug Flag indicating whether debug information should be printed into node log.
+         */
+        protected VisorCollectRunningQueriesJob(@Nullable Long arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Collection<VisorRunningQuery> run(@Nullable Long duration) throws IgniteException {
+            Collection<GridRunningQueryInfo> queries = ignite.context().query()
+                .runningQueries(duration != null ? duration : 0);
+
+            Collection<VisorRunningQuery> res = new ArrayList<>(queries.size());
+
+            long curTime = U.currentTimeMillis();
+
+            for (GridRunningQueryInfo qry : queries)
+                res.add(new VisorRunningQuery(qry.id(), qry.query(), qry.queryType(), qry.cache(),
+                    qry.startTime(), curTime - qry.startTime(),
+                    qry.cancelable(), qry.local()));
+
+            return res;
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java
new file mode 100644
index 0000000000000..fc6bc7a9222d4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
+
+/**
+ * Descriptor of running query.
+ */
+public class VisorRunningQuery implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private long id;
+
+    /** Query text. */
+    private String qry;
+
+    /** Query type. */
+    private GridCacheQueryType qryType;
+
+    /** Cache name for query. */
+    private String cache;
+
+    /** */
+    private long startTime;
+
+    /** */
+    private long duration;
+
+    /** */
+    private boolean cancellable;
+
+    /** */
+    private boolean loc;
+
+    /**
+     * @param id Query ID.
+     * @param qry Query text.
+     * @param qryType Query type.
+     * @param cache Cache where query was executed.
+     * @param startTime Query start time.
+     * @param duration Query current duration.
+     * @param cancellable {@code true} if query can be canceled.
+     * @param loc {@code true} if query is local.
+     */
+    public VisorRunningQuery(long id, String qry, GridCacheQueryType qryType, String cache,
+        long startTime, long duration,
+        boolean cancellable, boolean loc) {
+        this.id = id;
+        this.qry = qry;
+        this.qryType = qryType;
+        this.cache = cache;
+        this.startTime = startTime;
+        this.duration = duration;
+        this.cancellable = cancellable;
+        this.loc = loc;
+    }
+
+    /**
+     * @return Query ID.
+     */
+    public long getId() {
+        return id;
+    }
+
+    /**
+     * @return Query txt.
+     */
+    public String getQuery() {
+        return qry;
+    }
+
+    /**
+     * @return Query type.
+     */
+    public GridCacheQueryType getQueryType() {
+        return qryType;
+    }
+
+    /**
+     * @return Cache name.
+     */
+    public String getCache() {
+        return cache;
+    }
+
+    /**
+     * @return Query start time.
+     */
+    public long getStartTime() {
+        return startTime;
+    }
+
+    /**
+     * @return Query duration.
+     */
+    public long getDuration() {
+        return duration;
+    }
+
+    /**
+     * @return {@code true} if query can be cancelled.
+     */
+    public boolean isCancelable() {
+        return cancellable;
+    }
+
+    /**
+     * @return {@code true} if query is local.
+     */
+    public boolean isLocal() {
+        return loc;
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index cbf2ebde27056..62b47b85c6507 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -52,6 +52,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import javax.cache.Cache;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
@@ -79,6 +80,7 @@
 import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
 import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
@@ -172,6 +174,9 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_CLEANUP_PERIOD;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_THREAD_USAGE_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.getString;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.TEXT;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.FULLTEXT;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.GEO_SPATIAL;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.SORTED;
@@ -278,9 +283,15 @@ public class IgniteH2Indexing implements GridQueryIndexing {
     /** space name -> schema name */
     private final Map<String, String> space2schema = new ConcurrentHashMap8<>();
 
+    /** */
+    private AtomicLong qryIdGen;
+
     /** */
     private GridSpinBusyLock busyLock;
 
+    /** */
+    private final ConcurrentMap<Long, GridRunningQueryInfo> runs = new ConcurrentHashMap8<>();
+
     /** */
     private final ThreadLocal<ConnectionWrapper> connCache = new ThreadLocal<ConnectionWrapper>() {
         @Nullable @Override public ConnectionWrapper get() {
@@ -751,8 +762,19 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
         IndexingQueryFilter filters) throws IgniteCheckedException {
         TableDescriptor tbl = tableDescriptor(spaceName, type);
 
-        if (tbl != null && tbl.luceneIdx != null)
-            return tbl.luceneIdx.query(qry, filters);
+        if (tbl != null && tbl.luceneIdx != null) {
+            GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, TEXT, spaceName,
+                U.currentTimeMillis(), null, true);
+
+            try {
+                runs.put(run.id(), run);
+
+                return tbl.luceneIdx.query(qry, filters);
+            }
+            finally {
+                runs.remove(run.id());
+            }
+        }
 
         return new GridEmptyCloseableIterator<>();
     }
@@ -796,6 +818,11 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
 
                 GridH2QueryContext.set(ctx);
 
+                GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL_FIELDS,
+                    spaceName, U.currentTimeMillis(), cancel, true);
+
+                runs.putIfAbsent(run.id(), run);
+
                 try {
                     ResultSet rs = executeSqlQueryWithTimer(spaceName, stmt, conn, qry, params, timeout, cancel);
 
@@ -803,6 +830,8 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
                 }
                 finally {
                     GridH2QueryContext.clearThreadLocal();
+
+                    runs.remove(run.id());
                 }
             }
         };
@@ -1061,6 +1090,11 @@ public void setupConnection(Connection conn, boolean distributedJoins, boolean e
 
         GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter).distributedJoins(false));
 
+        GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL, spaceName,
+            U.currentTimeMillis(), null, true);
+
+        runs.put(run.id(), run);
+
         try {
             ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, null);
 
@@ -1068,6 +1102,8 @@ public void setupConnection(Connection conn, boolean distributedJoins, boolean e
         }
         finally {
             GridH2QueryContext.clearThreadLocal();
+
+            runs.remove(run.id());
         }
     }
 
@@ -1692,6 +1728,8 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
 
         this.busyLock = busyLock;
 
+        qryIdGen = new AtomicLong();
+
         if (SysProperties.serializeJavaObject) {
             U.warn(log, "Serialization of Java objects in H2 was enabled.");
 
@@ -1742,7 +1780,7 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
             marshaller = ctx.config().getMarshaller();
 
             mapQryExec = new GridMapQueryExecutor(busyLock);
-            rdcQryExec = new GridReduceQueryExecutor(busyLock);
+            rdcQryExec = new GridReduceQueryExecutor(qryIdGen, busyLock);
 
             mapQryExec.start(ctx, this);
             rdcQryExec.start(ctx, this);
@@ -2196,6 +2234,37 @@ private static List<IndexColumn> treeIndexColumns(List<IndexColumn> cols, IndexC
         return cols;
     }
 
+
+    /** {@inheritDoc} */
+    @Override public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+        Collection<GridRunningQueryInfo> res = new ArrayList<>();
+
+        res.addAll(runs.values());
+        res.addAll(rdcQryExec.longRunningQueries(duration));
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancelQueries(Collection<Long> queries) {
+        if (!F.isEmpty(queries)) {
+            for (Long qryId : queries) {
+                GridRunningQueryInfo run = runs.get(qryId);
+
+                if (run != null)
+                    run.cancel();
+            }
+
+            rdcQryExec.cancelQueries(queries);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancelAllQueries() {
+        for (Connection conn : conns)
+            U.close(conn, log);
+    }
+
     /**
      * Wrapper to store connection and flag is schema set or not.
      */
@@ -3086,10 +3155,4 @@ private void updateLastUsage() {
             lastUsage = U.currentTimeMillis();
         }
     }
-
-    /** {@inheritDoc} */
-    @Override public void cancelAllQueries() {
-        for (Connection conn : conns)
-            U.close(conn, log);
-    }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 7d43bf647ec1a..8284c45e0183d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -170,7 +170,7 @@ public static GridCacheTwoStepQuery split(
         qry = collectAllTables(qry, schemas, tbls);
 
         // Build resulting two step query.
-        GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(schemas, tbls);
+        GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(qry.getSQL(), schemas, tbls);
 
         // Map query will be direct reference to the original query AST.
         // Thus all the modifications will be performed on the original AST, so we should be careful when
@@ -954,4 +954,4 @@ private static GridSqlOperation op(GridSqlOperationType type, GridSqlElement lef
     private static GridSqlFunction function(GridSqlFunctionType type) {
         return new GridSqlFunction(type);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 40b1197317acd..3f886ee686d66 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -62,6 +62,7 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
 import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.h2.GridH2ResultSetIterator;
@@ -99,6 +100,7 @@
 import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
 import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REDUCE;
 
 /**
@@ -121,7 +123,7 @@ public class GridReduceQueryExecutor {
     private IgniteLogger log;
 
     /** */
-    private final AtomicLong reqIdGen = new AtomicLong();
+    private final AtomicLong qryIdGen;
 
     /** */
     private final ConcurrentMap<Long, QueryRun> runs = new ConcurrentHashMap8<>();
@@ -168,9 +170,11 @@ public class GridReduceQueryExecutor {
     };
 
     /**
+     * @param qryIdGen Query ID generator.
      * @param busyLock Busy lock.
      */
-    public GridReduceQueryExecutor(GridSpinBusyLock busyLock) {
+    public GridReduceQueryExecutor(AtomicLong qryIdGen, GridSpinBusyLock busyLock) {
+        this.qryIdGen = qryIdGen;
         this.busyLock = busyLock;
     }
 
@@ -494,11 +498,13 @@ public Iterator<List<?>> query(
                 }
             }
 
-            final long qryReqId = reqIdGen.incrementAndGet();
+            final long qryReqId = qryIdGen.incrementAndGet();
 
             final String space = cctx.name();
 
-            final QueryRun r = new QueryRun(h2.connectionForSpace(space), qry.mapQueries().size(), qry.pageSize());
+            final QueryRun r = new QueryRun(qryReqId, qry.originalSql(), space,
+                h2.connectionForSpace(space), qry.mapQueries().size(), qry.pageSize(),
+                U.currentTimeMillis(), cancel);
 
             AffinityTopologyVersion topVer = h2.readyTopologyVersion();
 
@@ -1303,10 +1309,46 @@ public void onDisconnected(IgniteFuture<?> reconnectFut) {
             e.getValue().disconnected(err);
     }
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of IDs and statements of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> longRunningQueries(long duration) {
+        Collection<GridRunningQueryInfo> res = new ArrayList<>();
+
+        long curTime = U.currentTimeMillis();
+
+        for (QueryRun run : runs.values()) {
+            if (run.qry.longQuery(curTime, duration))
+                res.add(run.qry);
+        }
+
+        return res;
+    }
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries IDs to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries) {
+        for (Long qryId : queries) {
+            QueryRun run = runs.get(qryId);
+
+            if (run != null)
+                run.qry.cancel();
+        }
+    }
+
     /**
      * Query run.
      */
     private static class QueryRun {
+        /** */
+        private final GridRunningQueryInfo qry;
+
         /** */
         private final List<GridMergeIndex> idxs;
 
@@ -1323,11 +1365,17 @@ private static class QueryRun {
         private final AtomicReference<Object> state = new AtomicReference<>();
 
         /**
+         * @param id Query ID.
+         * @param qry Query text.
+         * @param cache Cache where query was executed.
          * @param conn Connection.
          * @param idxsCnt Number of indexes.
          * @param pageSize Page size.
+         * @param startTime Start time.
+         * @param cancel Query cancel handler.
          */
-        private QueryRun(Connection conn, int idxsCnt, int pageSize) {
+        private QueryRun(Long id, String qry, String cache, Connection conn, int idxsCnt, int pageSize, long startTime, GridQueryCancel cancel) {
+            this.qry = new GridRunningQueryInfo(id, qry, SQL_FIELDS, cache, startTime, cancel, false);
             this.conn = (JdbcConnection)conn;
             this.idxs = new ArrayList<>(idxsCnt);
             this.pageSize = pageSize > 0 ? pageSize : GridCacheTwoStepQuery.DFLT_PAGE_SIZE;
@@ -1410,4 +1458,4 @@ private ExplicitPartitionsSpecializer(Map<ClusterNode,IntArray> partsMap) {
             return copy(msg, n, partsMap);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
index e47e893530384..66e7e4abf9cdb 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
@@ -17,15 +17,23 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.Query;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cache.query.annotations.QuerySqlFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -54,6 +62,7 @@ public class CacheSqlQueryValueCopySelfTest extends GridCommonAbstractTest {
 
         cc.setCopyOnRead(true);
         cc.setIndexedTypes(Integer.class, Value.class);
+        cc.setSqlFunctionClasses(TestSQLFunctions.class);
 
         cfg.setCacheConfiguration(cc);
 
@@ -72,7 +81,7 @@ public class CacheSqlQueryValueCopySelfTest extends GridCommonAbstractTest {
         IgniteCache<Integer, Value> cache = grid(0).cache(null);
 
         for (int i = 0; i < KEYS; i++)
-            cache.put(i, new Value("before"));
+            cache.put(i, new Value(i, "before-" + i));
     }
 
     /** {@inheritDoc} */
@@ -195,17 +204,148 @@ public void testLocalSqlFieldsQuery() {
         check(cache);
     }
 
-    /** */
-    private static class Value {
-        /** */
-        private String str;
+    /**
+     * Run specified query in separate thread.
+     *
+     * @param qry Query to execute.
+     */
+    private IgniteInternalFuture<?> runQueryAsync(final Query<?> qry) throws Exception {
+        return multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    log.info(">>> Query started");
+
+                    grid(0).cache(null).query(qry).getAll();
+
+                    log.info(">>> Query finished");
+                }
+                catch (Throwable e) {
+                    e.printStackTrace();
+                }
+            }
+        }, 1, "run-query");
+    }
 
-        /**
-         * @param str String.
-         */
-        public Value(String str) {
-            this.str = str;
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRunningSqlFieldsQuery() throws Exception {
+        IgniteInternalFuture<?> fut = runQueryAsync(new SqlFieldsQuery("select _val, sleep(1000) from Value limit 3"));
+
+        Thread.sleep(500);
+
+        GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+
+        SqlFieldsQuery qry = new SqlFieldsQuery("select _val, sleep(1000) from Value limit 3");
+        qry.setLocal(true);
+
+        fut = runQueryAsync(qry);
+
+        Thread.sleep(500);
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+    }
+
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRunningSqlQuery() throws Exception {
+        IgniteInternalFuture<?> fut = runQueryAsync(new SqlQuery<Integer, Value>(Value.class, "id > sleep(100)"));
+
+        Thread.sleep(500);
+
+        GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+
+        SqlQuery<Integer, Value> qry = new SqlQuery<>(Value.class, "id > sleep(100)");
+        qry.setLocal(true);
+
+        fut = runQueryAsync(qry);
+
+        Thread.sleep(500);
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+    }
+
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCancelingSqlFieldsQuery() throws Exception {
+        runQueryAsync(new SqlFieldsQuery("select * from (select _val, sleep(100) from Value limit 50)"));
+
+        Thread.sleep(500);
+
+        final GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        final Collection<GridRunningQueryInfo> finalQueries = queries;
+
+        for (GridRunningQueryInfo query : finalQueries)
+            qryProc.cancelQueries(Collections.singleton(query.id()));
+
+        int n = 100;
+
+        // Give cluster some time to cancel query and cleanup resources.
+        while (n > 0) {
+            Thread.sleep(100);
+
+            queries = qryProc.runningQueries(0);
+
+            if (queries.isEmpty())
+                break;
+
+            log.info(">>>> Wait for cancel: " + n);
+
+            n--;
         }
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
     }
 
     /**
@@ -218,9 +358,53 @@ private void check(IgniteCache<Integer, Value> cache) {
         for (Cache.Entry<Integer, Value> entry : cache) {
             cnt++;
 
-            assertEquals("before", entry.getValue().str);
+            assertEquals("before-" + entry.getKey(), entry.getValue().str);
         }
 
         assertEquals(KEYS, cnt);
     }
-}
\ No newline at end of file
+
+    /** */
+    private static class Value {
+        /** */
+        @QuerySqlField
+        private int id;
+
+        /** */
+        @QuerySqlField
+        private String str;
+
+        /**
+         * @param id ID.
+         * @param str String.
+         */
+        public Value(int id, String str) {
+            this.id = id;
+            this.str = str;
+        }
+    }
+
+    /**
+     * Utility class with custom SQL functions.
+     */
+    public static class TestSQLFunctions {
+        /**
+         * Sleep function to simulate long running queries.
+         *
+         * @param x Time to sleep.
+         * @return Return specified argument.
+         */
+        @QuerySqlFunction
+        public static long sleep(long x) {
+            if (x >= 0)
+                try {
+                    Thread.sleep(x);
+                }
+                catch (InterruptedException ignored) {
+                    // No-op.
+                }
+
+            return x;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
index 1f10593e61e0c..01fefa3ac95e3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
@@ -477,4 +477,4 @@ public int getStoreId() {
             return storeId;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
index ad8a7e34e82a7..814d0e0eef516 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
@@ -106,6 +106,9 @@ protected void startIndexing(IgniteH2Indexing spi) throws Exception {
         spi.registerCache(null, cacheCfg("B"));
     }
 
+    /**
+     * @param name Name.
+     */
     private CacheConfiguration cacheCfg(String name) {
         CacheConfiguration<?,?> cfg = new CacheConfiguration<>();
 
@@ -114,6 +117,7 @@ private CacheConfiguration cacheCfg(String name) {
         return cfg;
     }
 
+    /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         idx.stop();
 
@@ -182,6 +186,9 @@ private IgniteH2Indexing getIndexing() {
         return idx;
     }
 
+    /**
+     * @return {@code true} if OFF-HEAP mode should be tested.
+     */
     protected boolean offheap() {
         return false;
     }

From 2f57760dbb4fba948cd035498d2c7f71869c0665 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 17 Feb 2017 16:15:31 +0300
Subject: [PATCH 026/516] IGNITE-4624: Scan query optimization. This closes
 #1509.

---
 .../distributed/dht/GridDhtCacheAdapter.java  | 19 +++-
 .../cache/query/GridCacheQueryManager.java    | 97 ++++++++++---------
 2 files changed, 64 insertions(+), 52 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index dcd379a040bd0..be7fa5525deba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -1247,14 +1247,27 @@ public Iterator<Cache.Entry<K, V>> localEntriesIterator(final boolean primary,
         final boolean backup,
         final boolean keepBinary,
         final AffinityTopologyVersion topVer) {
+
+        return iterator(localEntriesIteratorEx(primary, backup, topVer), !keepBinary);
+    }
+
+    /**
+     * @param primary If {@code true} includes primary entries.
+     * @param backup If {@code true} includes backup entries.
+     * @param topVer Specified affinity topology version.
+     * @return Local entries iterator.
+     */
+    public Iterator<? extends GridCacheEntryEx> localEntriesIteratorEx(final boolean primary,
+        final boolean backup,
+        final AffinityTopologyVersion topVer) {
         assert primary || backup;
 
         if (primary && backup)
-            return iterator(entries().iterator(), !keepBinary);
+            return entries().iterator();
         else {
             final Iterator<GridDhtLocalPartition> partIt = topology().currentLocalPartitions().iterator();
 
-            Iterator<GridCacheMapEntry> it = new Iterator<GridCacheMapEntry>() {
+            return new Iterator<GridCacheMapEntry>() {
                 private GridCacheMapEntry next;
 
                 private Iterator<GridCacheMapEntry> curIt;
@@ -1311,8 +1324,6 @@ private void advance() {
                     while (partIt.hasNext());
                 }
             };
-
-            return iterator(it, !keepBinary);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index d64dff4b1a02e..14b1106eb5e1c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -1033,23 +1033,25 @@ private GridIterator<IgniteBiTuple<K, V>> swapIterator(
      * @throws GridDhtUnreservedPartitionException If failed to reserve partition.
      */
     private GridIterator<IgniteBiTuple<K, V>> onheapIterator(
-        GridCacheQueryAdapter<?> qry,
+        final GridCacheQueryAdapter<?> qry,
         AffinityTopologyVersion topVer,
         final IgniteBiPredicate<K, V> keyValFilter,
-        boolean backups,
+        final boolean backups,
         final ExpiryPolicy plc,
         final boolean locNode) throws GridDhtUnreservedPartitionException {
-        Iterator<K> keyIter;
+        Iterator<? extends GridCacheEntryEx> entryIter;
 
         GridDhtLocalPartition locPart = null;
 
         Integer part = qry.partition();
 
-        if (part == null || cctx.isLocal()) {
-            // Performance optimization.
-            if (locNode && plc == null && !cctx.isLocal()) {
-                GridDhtCacheAdapter<K, V> cache = cctx.isNear() ? cctx.near().dht() : cctx.dht();
+        if (cctx.isLocal())
+            entryIter = cctx.local().allEntries().iterator();
+        else if (part == null) {
+            GridDhtCacheAdapter<K, V> cache = cctx.isNear() ? cctx.near().dht() : cctx.dht();
 
+            // Performance optimization.
+            if (locNode && plc == null) {
                 final Iterator<Cache.Entry<K, V>> iter = cache.localEntriesIterator(true,
                     backups, cache.context().keepBinary(), topVer);
 
@@ -1099,12 +1101,10 @@ private void advance() {
                 };
             }
 
-            IgniteInternalCache<K, V> keepBinaryCache = cctx.cache().keepBinary();
-
-            keyIter = backups ? keepBinaryCache.keySetx().iterator() : keepBinaryCache.primaryKeySet().iterator();
+            entryIter = cache.localEntriesIteratorEx(true, backups, topVer);
         }
         else if (part < 0 || part >= cctx.affinity().partitions())
-            keyIter = new GridEmptyIterator<>();
+            return new GridEmptyIterator<>();
         else {
             final GridDhtCacheAdapter dht = cctx.isNear() ? cctx.near().dht() : cctx.dht();
 
@@ -1115,28 +1115,12 @@ else if (part < 0 || part >= cctx.affinity().partitions())
                 throw new GridDhtUnreservedPartitionException(part, cctx.affinity().affinityTopologyVersion(),
                     "Partition can not be reserved.");
 
-            final GridDhtLocalPartition locPart0 = locPart;
-
-            keyIter = new Iterator<K>() {
-                private Iterator<KeyCacheObject> iter0 = locPart0.keySet().iterator();
-
-                @Override public boolean hasNext() {
-                    return iter0.hasNext();
-                }
-
-                @Override public K next() {
-                    return (K)iter0.next();
-                }
-
-                @Override public void remove() {
-                    iter0.remove();
-                }
-            };
+            entryIter = locPart.allEntries().iterator();
         }
 
         final GridDhtLocalPartition locPart0 = locPart;
 
-        return new PeekValueExpiryAwareIterator(keyIter, plc, topVer, keyValFilter, qry.keepBinary(), locNode, true) {
+        return new PeekValueExpiryAwareIterator(entryIter, plc, topVer, keyValFilter, qry.keepBinary(), locNode, true) {
             @Override protected void onClose() {
                 super.onClose();
 
@@ -1263,18 +1247,20 @@ private GridIterator<IgniteBiTuple<K, V>> scanExpiryIterator(
         ExpiryPolicy expPlc,
         final boolean keepBinary,
         boolean locNode) {
-        Iterator<K> keyIter = new Iterator<K>() {
+        Iterator<? extends GridCacheEntryEx> keyIter = new Iterator<GridCacheEntryEx>() {
             /** {@inheritDoc} */
             @Override public boolean hasNext() {
                 return it.hasNext();
             }
 
             /** {@inheritDoc} */
-            @Override public K next() {
+            @Override public GridCacheEntryEx next() {
                 try {
                     KeyCacheObject key = cctx.toCacheKeyObject(it.next().getKey());
 
-                    return (K)cctx.unwrapBinaryIfNeeded(key, keepBinary);
+                    final GridCacheEntryEx entryEx = cctx.cache().entryEx(key);
+
+                    return entryEx;
                 }
                 catch (IgniteCheckedException e) {
                     throw new IgniteException(e);
@@ -2189,8 +2175,8 @@ public QueryMetrics metrics() {
     }
 
     /**
-     * Gets cache queries detailed metrics.
-     * Detail metrics could be enabled by setting non-zero value via {@link CacheConfiguration#setQueryDetailMetricsSize(int)}
+     * Gets cache queries detailed metrics. Detail metrics could be enabled by setting non-zero value via {@link
+     * CacheConfiguration#setQueryDetailMetricsSize(int)}
      *
      * @return Cache queries metrics aggregated by query type and query text.
      */
@@ -3091,8 +3077,10 @@ private CompoundIterator(List<GridIterator<T>> iters) {
     private abstract static class CachedResult<R> extends GridFutureAdapter<IgniteSpiCloseableIterator<R>> {
         /** Absolute position of each recipient. */
         private final Map<Object, QueueIterator> recipients = new GridLeanMap<>(1);
+
         /** */
         private CircularQueue<R> queue;
+
         /** */
         private int pruned;
 
@@ -3529,10 +3517,10 @@ private class PeekValueExpiryAwareIterator extends GridCloseableIteratorAdapter<
         private IgniteCacheExpiryPolicy expiryPlc;
 
         /** */
-        private Iterator<K> keyIt;
+        private Iterator<? extends GridCacheEntryEx> entryIt;
 
         /**
-         * @param keyIt Key iterator.
+         * @param entryIter Key iterator.
          * @param plc Expiry policy.
          * @param topVer Topology version.
          * @param keyValFilter Key-value filter.
@@ -3540,8 +3528,8 @@ private class PeekValueExpiryAwareIterator extends GridCloseableIteratorAdapter<
          * @param locNode Local node.
          * @param heapOnly Heap only.
          */
-        private PeekValueExpiryAwareIterator(
-            Iterator<K> keyIt,
+        PeekValueExpiryAwareIterator(
+            Iterator<? extends GridCacheEntryEx> entryIter,
             ExpiryPolicy plc,
             AffinityTopologyVersion topVer,
             IgniteBiPredicate<K, V> keyValFilter,
@@ -3549,7 +3537,7 @@ private PeekValueExpiryAwareIterator(
             boolean locNode,
             boolean heapOnly
         ) {
-            this.keyIt = keyIt;
+            this.entryIt = entryIter;
             this.plc = plc;
             this.topVer = topVer;
             this.keyValFilter = keyValFilter;
@@ -3593,15 +3581,27 @@ private PeekValueExpiryAwareIterator(
         private void advance() {
             IgniteBiTuple<K, V> next0 = null;
 
-            while (keyIt.hasNext()) {
+            while (entryIt.hasNext()) {
                 next0 = null;
 
-                K key = keyIt.next();
+                GridCacheEntryEx entry = entryIt.next();
+
+                if (entry.deleted())
+                    continue;
 
+                KeyCacheObject key = entry.key();
                 CacheObject val;
 
                 try {
-                    val = value(key);
+                    if (heapOnly)
+                        val = entry.peek(true, false, false, expiryPlc);
+                    else
+                        val = value(entry, entry.key());
+                }
+                catch (GridCacheEntryRemovedException ignore) {
+                    assert heapOnly;
+
+                    continue;
                 }
                 catch (IgniteCheckedException e) {
                     if (log.isDebugEnabled())
@@ -3664,23 +3664,24 @@ private void sendTtlUpdate() {
         }
 
         /**
+         * @param entry Entry.
          * @param key Key.
          * @return Value.
          * @throws IgniteCheckedException If failed to peek value.
          */
-        private CacheObject value(K key) throws IgniteCheckedException {
+        private CacheObject value(GridCacheEntryEx entry, KeyCacheObject key) throws IgniteCheckedException {
             while (true) {
                 try {
-                    GridCacheEntryEx entry = heapOnly ? cache.peekEx(key) : cache.entryEx(key);
+                    if (entry == null)
+                        entry = cache.entryEx(key);
 
-                    if (expiryPlc != null && !heapOnly)
+                    if (expiryPlc != null)
                         entry.unswap();
 
-                    return entry != null ? entry.peek(true, !heapOnly, !heapOnly, topVer, expiryPlc) : null;
+                    return entry.peek(true, true, true, topVer, expiryPlc);
                 }
                 catch (GridCacheEntryRemovedException ignore) {
-                    if (heapOnly)
-                        return null;
+                    entry = null;
                 }
             }
         }

From c0e2df26f056cd11690d821146f05e3fd938906e Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 20 Feb 2017 11:17:35 +0300
Subject: [PATCH 027/516] IGNITE-3429 - Rollback due to broken compilation

---
 .../Hibernate5CacheKeyTypeConfiguration.java  | 52 -------------------
 .../HibernateCacheKeyTypeConfiguration.java   | 51 ------------------
 2 files changed, 103 deletions(-)
 delete mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
 delete mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java

diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
deleted file mode 100644
index 886f69b2500ba..0000000000000
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/Hibernate5CacheKeyTypeConfiguration.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.hibernate.config;
-
-import java.util.Objects;
-import org.apache.ignite.binary.BinaryAbstractIdentityResolver;
-import org.apache.ignite.binary.BinaryIdentityResolver;
-import org.apache.ignite.binary.BinaryObject;
-import org.apache.ignite.binary.BinaryTypeConfiguration;
-
-/**
- * This configuration provides correct {@link BinaryIdentityResolver} implementation
- * for Hibernate CacheKey class can be used as a key object.
- *
- * Note: for Hibernate version < 5.0 {@link HibernateCacheKeyTypeConfiguration} should be used.
-
- */
-public class Hibernate5CacheKeyTypeConfiguration extends BinaryTypeConfiguration {
-
-    /** {@inheritDoc} */
-    public Hibernate5CacheKeyTypeConfiguration() {
-        super("org.hibernate.cache.internal.CacheKeyImplementation");
-
-        setIdentityResolver(new BinaryAbstractIdentityResolver() {
-            @Override protected int hashCode0(BinaryObject obj) {
-                return obj.field("id").hashCode();
-            }
-
-            @Override protected boolean equals0(BinaryObject o1, BinaryObject o2) {
-                Object obj0 = o1.field("id");
-                Object obj1 = o2.field("id");
-
-                return Objects.equals(obj0, obj1);
-            }
-        });
-    }
-}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java
deleted file mode 100644
index c54292e6f40e6..0000000000000
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/config/HibernateCacheKeyTypeConfiguration.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.hibernate.config;
-
-import java.util.Objects;
-import org.apache.ignite.binary.BinaryAbstractIdentityResolver;
-import org.apache.ignite.binary.BinaryIdentityResolver;
-import org.apache.ignite.binary.BinaryObject;
-import org.apache.ignite.binary.BinaryTypeConfiguration;
-
-/**
- * This configuration provides correct {@link BinaryIdentityResolver} implementation
- * for Hibernate CacheKey class can be used as a key object.
- *
- * Note: for Hibernate version >= 5.0 {@link Hibernate5CacheKeyTypeConfiguration} should be used.
- */
-public class HibernateCacheKeyTypeConfiguration extends BinaryTypeConfiguration {
-
-    /** {@inheritDoc} */
-    public HibernateCacheKeyTypeConfiguration() {
-        super("org.hibernate.cache.spi.CacheKey");
-
-        setIdentityResolver(new BinaryAbstractIdentityResolver() {
-            @Override protected int hashCode0(BinaryObject obj) {
-                return obj.field("key").hashCode();
-            }
-
-            @Override protected boolean equals0(BinaryObject o1, BinaryObject o2) {
-                Object obj0 = o1.field("key");
-                Object obj1 = o2.field("key");
-
-                return Objects.equals(obj0, obj1);
-            }
-        });
-    }
-}

From c849534df6583043d6a1d4f454cb981a20896d1a Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Tue, 21 Feb 2017 10:08:55 +0700
Subject: [PATCH 028/516] IGNITE-4472 Added user activities in Web Console.

(cherry picked from commit 26ee9c2865648118da97ee8ef84df990359edb96)
---
 .../frontend/app/modules/demo/Demo.module.js       | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js
index a3700ca3ed141..bd759df552c23 100644
--- a/modules/web-console/frontend/app/modules/demo/Demo.module.js
+++ b/modules/web-console/frontend/app/modules/demo/Demo.module.js
@@ -23,22 +23,26 @@ angular
 .module('ignite-console.demo', [
     'ignite-console.socket'
 ])
-.config(['$stateProvider', ($stateProvider) => {
+.config(['$stateProvider', 'AclRouteProvider', ($stateProvider, AclRoute) => {
     $stateProvider
         .state('demo', {
             abstract: true,
+            url: '/demo',
             template: '<ui-view></ui-view>'
         })
         .state('demo.resume', {
-            url: '/demo',
+            url: '/resume',
+            onEnter: AclRoute.checkAccess('demo'),
             controller: ['$state', ($state) => {
                 $state.go('base.configuration.clusters');
             }],
             metaTags: {
+                title: 'Demo resume'
             }
         })
         .state('demo.reset', {
-            url: '/demo/reset',
+            url: '/reset',
+            onEnter: AclRoute.checkAccess('demo'),
             controller: ['$state', '$http', 'IgniteMessages', ($state, $http, Messages) => {
                 $http.post('/api/v1/demo/reset')
                     .then(() => $state.go('base.configuration.clusters'))
@@ -48,7 +52,9 @@ angular
                         Messages.showError(res);
                     });
             }],
-            metaTags: {}
+            metaTags: {
+                title: 'Demo reset'
+            }
         });
 }])
 .provider('Demo', ['$stateProvider', '$httpProvider', 'igniteSocketFactoryProvider', function($state, $http, socketFactory) {

From 9df5e94d5cf14ddd55e29b81989177a7798f7e1a Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 21 Feb 2017 15:34:59 +0300
Subject: [PATCH 029/516] IGNITE-4671 - FairAffinityFunction fails on node
 restart with backupFilter set and no backups

---
 .../affinity/fair/FairAffinityFunction.java   |  33 ++---
 .../fair/FairAffinityNodesRestart.java        | 130 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite.java      |   2 +
 3 files changed, 149 insertions(+), 16 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityNodesRestart.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
index cf1cb02117c75..7acb5b429e693 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
@@ -911,37 +911,37 @@ private boolean isAssignable(int part, int tier, final ClusterNode node, boolean
             if (exclNeighbors)
                 return allowNeighbors || !neighborsContainPartition(node, part);
             else if (affinityBackupFilter != null) {
-                List<ClusterNode> assigment = assignments.get(part);
+                List<ClusterNode> assignment = assignments.get(part);
 
-                assert assigment.size() > 0;
+                if (assignment.isEmpty())
+                    return true;
 
                 List<ClusterNode> newAssignment;
 
                 if (tier == 0) {
-                    for (int t = 1; t < assigment.size(); t++) {
-                        newAssignment = new ArrayList<>(assigment.size() - 1);
+                    for (int t = 1; t < assignment.size(); t++) {
+                        newAssignment = new ArrayList<>(assignment.size() - 1);
 
                         newAssignment.add(node);
 
                         if (t != 1)
-                            newAssignment.addAll(assigment.subList(1, t));
+                            newAssignment.addAll(assignment.subList(1, t));
 
-                        if (t + 1 < assigment.size())
-                            newAssignment.addAll(assigment.subList(t + 1, assigment.size()));
+                        if (t + 1 < assignment.size())
+                            newAssignment.addAll(assignment.subList(t + 1, assignment.size()));
 
-                        if (!affinityBackupFilter.apply(assigment.get(t), newAssignment))
+                        if (!affinityBackupFilter.apply(assignment.get(t), newAssignment))
                             return false;
-
                     }
 
                     return true;
                 }
-                else if (tier < assigment.size()) {
-                    newAssignment = new ArrayList<>(assigment.size() - 1);
+                else if (tier < assignment.size()) {
+                    newAssignment = new ArrayList<>(assignment.size() - 1);
 
                     int i = 0;
 
-                    for (ClusterNode assignmentNode: assigment) {
+                    for (ClusterNode assignmentNode: assignment) {
                         if (i != tier)
                             newAssignment.add(assignmentNode);
 
@@ -949,17 +949,18 @@ else if (tier < assigment.size()) {
                     }
                 }
                 else
-                    newAssignment = assigment;
+                    newAssignment = assignment;
 
                 return affinityBackupFilter.apply(node, newAssignment);
             }
             else if (backupFilter != null) {
                 if (tier == 0) {
-                    List<ClusterNode> assigment = assignments.get(part);
+                    List<ClusterNode> assignment = assignments.get(part);
 
-                    assert assigment.size() > 0;
+                    if (assignment.isEmpty())
+                        return true;
 
-                    List<ClusterNode> backups = assigment.subList(1, assigment.size());
+                    List<ClusterNode> backups = assignment.subList(1, assignment.size());
 
                     return !F.exist(backups, new IgnitePredicate<ClusterNode>() {
                         @Override public boolean apply(ClusterNode n) {
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityNodesRestart.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityNodesRestart.java
new file mode 100644
index 0000000000000..37f1bfb211297
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityNodesRestart.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.affinity.fair;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.P2;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests that FairAffinityFunction doesn't throw exception on nodes restart,
+ * with backup filter set and 0 cache backups.
+ */
+public class FairAffinityNodesRestart extends GridCommonAbstractTest {
+    /** */
+    private final static P2<ClusterNode, ClusterNode> BACKUP_FILTER = new P2<ClusterNode, ClusterNode>() {
+        @Override public boolean apply(ClusterNode node, ClusterNode node2) {
+            return true;
+        }
+    };
+
+    /** */
+    private final static P2<ClusterNode, List<ClusterNode>> AFF_BACKUP_FILTER = new P2<ClusterNode, List<ClusterNode>>() {
+        @Override public boolean apply(ClusterNode node, List<ClusterNode> nodes) {
+            return true;
+        }
+    };
+
+    /** */
+    private boolean affBackup;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = new CacheConfiguration("fair-cache");
+
+        FairAffinityFunction aff = new FairAffinityFunction(32);
+
+        if (!affBackup)
+            aff.setBackupFilter(BACKUP_FILTER);
+        else
+            aff.setAffinityBackupFilter(AFF_BACKUP_FILTER);
+
+        ccfg.setAffinity(aff);
+        ccfg.setBackups(0);
+        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /**
+     * @param idx Node index.
+     * @return Future.
+     */
+    private IgniteInternalFuture<IgniteEx> startAsyncGrid(final int idx) {
+        return GridTestUtils.runAsync(new Callable<IgniteEx>() {
+            @Override public IgniteEx call() throws Exception {
+                return startGrid(idx);
+            }
+        });
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBackupFilter() throws Exception {
+        affBackup = false;
+
+        check();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAffinityBackupFilter() throws Exception {
+        affBackup = true;
+
+        check();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void check() throws Exception {
+        for (int i = 0; i < 2; i++) {
+            IgniteInternalFuture<IgniteEx> fut0 = startAsyncGrid(0);
+            IgniteInternalFuture<IgniteEx> fut1 = startAsyncGrid(1);
+            IgniteInternalFuture<IgniteEx> fut2 = startAsyncGrid(2);
+
+            IgniteEx ignite = fut0.get();
+            fut1.get();
+            fut2.get();
+
+            IgniteCache<Integer, String> cache = ignite.cache("fair-cache");
+
+            for (int j = 0; j < 100; j++)
+                cache.put(i, String.valueOf(i));
+
+            stopGrid(0);
+            stopGrid(1);
+            stopGrid(2);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 092d95eff12bd..a24f020103e54 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.cache.affinity.fair.FairAffinityDynamicCacheSelfTest;
 import org.apache.ignite.cache.affinity.fair.FairAffinityFunctionNodesSelfTest;
 import org.apache.ignite.cache.affinity.fair.FairAffinityFunctionSelfTest;
+import org.apache.ignite.cache.affinity.fair.FairAffinityNodesRestart;
 import org.apache.ignite.cache.affinity.local.LocalAffinityFunctionTest;
 import org.apache.ignite.cache.store.GridCacheBalancingStoreSelfTest;
 import org.apache.ignite.cache.store.GridCacheLoadOnlyStoreAdapterSelfTest;
@@ -231,6 +232,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(FairAffinityFunctionNodesSelfTest.class);
         suite.addTestSuite(FairAffinityFunctionSelfTest.class);
         suite.addTestSuite(FairAffinityDynamicCacheSelfTest.class);
+        suite.addTestSuite(FairAffinityNodesRestart.class);
         suite.addTestSuite(GridCacheAffinityBackupsSelfTest.class);
         suite.addTestSuite(IgniteCacheAffinitySelfTest.class);
         suite.addTestSuite(AffinityClientNodeSelfTest.class);

From 9fcb3e74f91c8497b7b1358cdff40950cdf5c568 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 28 Feb 2017 16:05:06 +0300
Subject: [PATCH 030/516] IGNITE-4740 - Fix. Service could be
 deployed/undeployed twice on concurrent cancel and discovery event.

---
 .../cache/DynamicCacheChangeBatch.java        |  14 ++
 .../service/GridServiceProcessor.java         |  49 ++---
 .../GridServiceContinuousQueryRedeploy.java   | 167 ++++++++++++++++++
 .../testsuites/IgniteKernalSelfTestSuite.java |   2 +
 4 files changed, 208 insertions(+), 24 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceContinuousQueryRedeploy.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
index 4dcff9b8bc2f2..a2500633a1ccf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
@@ -113,6 +113,20 @@ public boolean clientReconnect() {
         return clientReconnect;
     }
 
+    /**
+     * @return {@code True} if request should trigger partition exchange.
+     */
+    public boolean exchangeNeeded() {
+        if (reqs != null) {
+            for (DynamicCacheChangeRequest req : reqs) {
+                if (req.exchangeNeeded())
+                    return true;
+            }
+        }
+
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(DynamicCacheChangeBatch.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 3690f357f1e5e..4eeafed421214 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -65,10 +65,12 @@
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.internal.processors.cache.CacheIteratorConverter;
+import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.query.CacheQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.internal.processors.continuous.AbstractContinuousMessage;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridEmptyIterator;
@@ -1468,19 +1470,7 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
         else {
             String name = e.getKey().name();
 
-            svcName.set(name);
-
-            Collection<ServiceContextImpl> ctxs;
-
-            synchronized (locSvcs) {
-                ctxs = locSvcs.remove(name);
-            }
-
-            if (ctxs != null) {
-                synchronized (ctxs) {
-                    cancel(ctxs, ctxs.size());
-                }
-            }
+            undeploy(name);
 
             // Finish deployment futures if undeployment happened.
             GridFutureAdapter<?> fut = depFuts.remove(name);
@@ -1586,6 +1576,12 @@ private class TopologyListener implements GridLocalEventListener {
                         if (!((CacheAffinityChangeMessage)msg).exchangeNeeded())
                             return;
                     }
+                    else if (msg instanceof DynamicCacheChangeBatch) {
+                        if (!((DynamicCacheChangeBatch)msg).exchangeNeeded())
+                            return;
+                    }
+                    else
+                        return;
                 }
                 else
                     topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
@@ -1771,21 +1767,26 @@ private void processAssignment(CacheEntryEvent<GridServiceAssignmentsKey, GridSe
             }
         }
         // Handle undeployment.
-        else {
-            String name = e.getKey().name();
+        else
+            undeploy(e.getKey().name());
+    }
 
-            svcName.set(name);
 
-            Collection<ServiceContextImpl> ctxs;
+    /**
+     * @param name Name.
+     */
+    private void undeploy(String name) {
+        svcName.set(name);
 
-            synchronized (locSvcs) {
-                ctxs = locSvcs.remove(name);
-            }
+        Collection<ServiceContextImpl> ctxs;
 
-            if (ctxs != null) {
-                synchronized (ctxs) {
-                    cancel(ctxs, ctxs.size());
-                }
+        synchronized (locSvcs) {
+            ctxs = locSvcs.remove(name);
+        }
+
+        if (ctxs != null) {
+            synchronized (ctxs) {
+                cancel(ctxs, ctxs.size());
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceContinuousQueryRedeploy.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceContinuousQueryRedeploy.java
new file mode 100644
index 0000000000000..1a9ef3a084e37
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceContinuousQueryRedeploy.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryListenerException;
+import javax.cache.event.CacheEntryUpdatedListener;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests whether concurrent service cancel and registering ContinuousQuery doesn't causes
+ * service redeployment.
+ */
+public class GridServiceContinuousQueryRedeploy extends GridCommonAbstractTest {
+    /** */
+    private static final String CACHE_NAME = "TEST_CACHE";
+
+    /** */
+    private static final String TEST_KEY = "TEST_KEY";
+
+    /** */
+    private static final String SERVICE_NAME = "service1";
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testServiceRedeploymentAfterCancel() throws Exception {
+        final Ignite ignite = startGrid(0);
+
+        final IgniteCache<Object, Object> managementCache = ignite.getOrCreateCache(CACHE_NAME);
+
+        final ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+        final List<Object> evts = Collections.synchronizedList(new ArrayList<>());
+
+        qry.setLocalListener(new CacheEntryUpdatedListener<Object, Object>() {
+            @Override public void onUpdated(
+                Iterable<CacheEntryEvent<?, ?>> iterable) throws CacheEntryListenerException {
+                for (CacheEntryEvent<?, ?> event : iterable)
+                    evts.add(event);
+            }
+        });
+
+        int iterations = 100;
+
+        while (iterations-- > 0) {
+            QueryCursor quorumCursor = managementCache.query(qry);
+
+            IgniteInternalFuture<?> fut1 = GridTestUtils.runAsync(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    System.out.println("Deploy " + SERVICE_NAME);
+                    deployService(ignite);
+
+                    return null;
+                }
+            });
+
+            IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    System.out.println("Undeploy " + SERVICE_NAME);
+                    ignite.services().cancel(SERVICE_NAME);
+
+                    return null;
+                }
+            });
+
+            fut1.get();
+            fut2.get();
+
+            U.sleep(100);
+
+            assert evts.size() <= 1 : evts.size();
+
+            ignite.services().cancel("service1");
+
+            evts.clear();
+
+            quorumCursor.close();
+        }
+
+    }
+
+    /**
+     * @param ignite Ignite.
+     */
+    private void deployService(final Ignite ignite) {
+        ServiceConfiguration svcCfg = new ServiceConfiguration();
+
+        svcCfg.setService(new ManagementService());
+        svcCfg.setName(SERVICE_NAME);
+        svcCfg.setTotalCount(1);
+        svcCfg.setMaxPerNodeCount(1);
+        svcCfg.setNodeFilter(new IgnitePredicate<ClusterNode>() {
+            @Override public boolean apply(ClusterNode node) {
+                return !node.isClient();
+            }
+        });
+
+        ignite.services().deploy(svcCfg);
+    }
+
+    /**
+     *
+     */
+    public static class ManagementService implements Service {
+        /** */
+        private final String name = UUID.randomUUID().toString();
+
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+            System.out.println(name + " shutdown.");
+        }
+
+        /** {@inheritDoc} */
+        @Override public synchronized void init(ServiceContext ctx) throws Exception {
+            System.out.println(name + " initializing.");
+
+            ignite.cache(CACHE_NAME).put(TEST_KEY, name + " init");
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+            // No-op
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 350b7152c36d9..59777023e8682 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -50,6 +50,7 @@
 import org.apache.ignite.internal.processors.cluster.GridUpdateNotifierSelfTest;
 import org.apache.ignite.internal.processors.port.GridPortProcessorSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceClientNodeTest;
+import org.apache.ignite.internal.processors.service.GridServiceContinuousQueryRedeploy;
 import org.apache.ignite.internal.processors.service.GridServicePackagePrivateSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceProcessorMultiNodeConfigSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceProcessorMultiNodeSelfTest;
@@ -143,6 +144,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridServiceProxyClientReconnectSelfTest.class);
         suite.addTestSuite(IgniteServiceReassignmentTest.class);
         suite.addTestSuite(IgniteServiceProxyTimeoutInitializedTest.class);
+        suite.addTestSuite(GridServiceContinuousQueryRedeploy.class);
 
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class);

From 231984d04d8391ccafb33ce995d942a9033cae44 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 1 Mar 2017 09:49:08 +0700
Subject: [PATCH 031/516] Web console: removed legacy backend port.

(cherry picked from commit 5456caf)
---
 modules/web-console/backend/app/settings.js |  8 +-------
 modules/web-console/backend/index.js        | 10 ----------
 2 files changed, 1 insertion(+), 17 deletions(-)

diff --git a/modules/web-console/backend/app/settings.js b/modules/web-console/backend/app/settings.js
index a79572e6989f0..94f3d90f58664 100644
--- a/modules/web-console/backend/app/settings.js
+++ b/modules/web-console/backend/app/settings.js
@@ -51,13 +51,7 @@ module.exports.factory = function(nconf, fs) {
 
     return {
         agent: {
-            dists: 'agent_dists',
-            port: _normalizePort(nconf.get('agentServer:port') || 3002),
-            SSLOptions: nconf.get('agentServer:ssl') && {
-                key: fs.readFileSync(nconf.get('agentServer:key')),
-                cert: fs.readFileSync(nconf.get('agentServer:cert')),
-                passphrase: nconf.get('agentServer:keyPassphrase')
-            }
+            dists: 'agent_dists'
         },
         server: {
             port: _normalizePort(nconf.get('server:port') || 3000),
diff --git a/modules/web-console/backend/index.js b/modules/web-console/backend/index.js
index 9eeff3516c2ee..27d7298dc5881 100644
--- a/modules/web-console/backend/index.js
+++ b/modules/web-console/backend/index.js
@@ -86,16 +86,6 @@ Promise.all([injector('settings'), injector('app'), injector('agent-manager'), i
         agentMgr.attach(server);
         browserMgr.attach(server);
 
-        // Start legacy agent server.
-        const agentServer = settings.agent.SSLOptions
-            ? https.createServer(settings.agent.SSLOptions) : http.createServer();
-
-        agentServer.listen(settings.agent.port);
-        agentServer.on('error', _onError.bind(null, settings.agent.port));
-        agentServer.on('listening', _onListening.bind(null, agentServer.address()));
-
-        agentMgr.attachLegacy(agentServer);
-
         // Used for automated test.
         if (process.send)
             process.send('running');

From 573624796b171b2420b87657598198f40a91f6bb Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@gridgain.com>
Date: Wed, 1 Mar 2017 22:09:40 +0700
Subject: [PATCH 032/516] Implemented support for enforce join order flag.
 (cherry picked from commit a7f77d4)

---
 .../internal/visor/query/VisorQueryArgV3.java | 51 +++++++++++++++++++
 .../internal/visor/query/VisorQueryJob.java   |  6 +--
 .../resources/META-INF/classnames.properties  |  1 +
 3 files changed, 55 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java
new file mode 100644
index 0000000000000..f32c00a79484b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+/**
+ * Arguments for {@link VisorQueryTask}.
+ */
+public class VisorQueryArgV3 extends VisorQueryArgV2 {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Enforce join order flag. */
+    private final boolean enforceJoinOrder;
+
+    /**
+     * @param cacheName Cache name for query.
+     * @param qryTxt Query text.
+     * @param distributedJoins If {@code true} then distributed joins enabled.
+     * @param enforceJoinOrder If {@code true} then enforce join order.
+     * @param loc Flag whether to execute query locally.
+     * @param pageSize Result batch size.
+     */
+    public VisorQueryArgV3(String cacheName, String qryTxt,
+        boolean distributedJoins, boolean enforceJoinOrder, boolean loc, int pageSize) {
+        super(cacheName, qryTxt, distributedJoins, loc, pageSize);
+
+        this.enforceJoinOrder = enforceJoinOrder;
+    }
+
+    /**
+     * @return Enforce join order flag.
+     */
+    public boolean enforceJoinOrder() {
+        return enforceJoinOrder;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
index c66b2dda8d22c..1ac90ad6a3491 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
@@ -131,9 +131,8 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 if (scanWithFilter) {
                     boolean caseSensitive = qryTxt.startsWith(SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE);
 
-                    String ptrn = caseSensitive
-                        ? qryTxt.substring(SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE.length())
-                        : qryTxt.substring(SCAN_CACHE_WITH_FILTER.length());
+                    String ptrn = qryTxt.substring(
+                        caseSensitive ? SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE.length() : SCAN_CACHE_WITH_FILTER.length());
 
                     filter = new VisorQueryScanSubstringFilter(caseSensitive, ptrn);
                 }
@@ -162,6 +161,7 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 qry.setPageSize(arg.pageSize());
                 qry.setLocal(arg.local());
                 qry.setDistributedJoins(arg instanceof VisorQueryArgV2 && ((VisorQueryArgV2)arg).distributedJoins());
+                qry.setEnforceJoinOrder(arg instanceof VisorQueryArgV3 && ((VisorQueryArgV3)arg).enforceJoinOrder());
 
                 long start = U.currentTimeMillis();
 
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 8a6dc6603c4f1..db486a5600695 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -1792,6 +1792,7 @@ org.apache.ignite.internal.visor.node.VisorSpisConfiguration
 org.apache.ignite.internal.visor.node.VisorTransactionConfiguration
 org.apache.ignite.internal.visor.query.VisorQueryArg
 org.apache.ignite.internal.visor.query.VisorQueryArgV2
+org.apache.ignite.internal.visor.query.VisorQueryArgV3
 org.apache.ignite.internal.visor.query.VisorQueryCleanupTask
 org.apache.ignite.internal.visor.query.VisorQueryCleanupTask$VisorQueryCleanupJob
 org.apache.ignite.internal.visor.query.VisorQueryField

From a30183ac821507fbdaa6f0cc2c6ef25ca2677867 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Mon, 20 Feb 2017 18:23:33 +0700
Subject: [PATCH 033/516] IGNITE-4717 Fixed hangs in VisorCacheClearTask.
 (cherry picked from commit 76f3060)

---
 .../visor/cache/VisorCacheClearTask.java      | 88 +++++--------------
 .../visor/compute/VisorGatewayTask.java       | 30 ++++++-
 2 files changed, 49 insertions(+), 69 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
index 1f1a6fb106bd3..0c8476fa91b10 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.internal.visor.cache;
 
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.internal.processors.task.GridInternal;
@@ -26,7 +25,6 @@
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.resources.JobContextResource;
@@ -90,17 +88,11 @@ private VisorCacheClearJob(String cacheName, boolean debug) {
         }
 
         /**
-         * @param subJob Sub job to execute asynchronously.
+         * @param fut Future for asynchronous cache operation.
          * @param idx Index.
          * @return {@code true} If subJob was not completed and this job should be suspended.
          */
-        private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
-            IgniteCompute compute = ignite.compute(ignite.cluster().forCacheNodes(cacheName)).withAsync();
-
-            compute.call(subJob);
-
-            IgniteFuture<Integer> fut = compute.future();
-
+        private boolean callAsync(IgniteFuture<Integer> fut, int idx) {
             futs[idx] = fut;
 
             if (fut.isDone())
@@ -119,16 +111,28 @@ private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
                 futs = new IgniteFuture[3];
 
             if (futs[0] == null || futs[1] == null || futs[2] == null) {
-                IgniteCache cache = ignite.cache(cacheName);
+                IgniteCache cache = ignite.cache(cacheName).withAsync();
+
+                if (futs[0] == null) {
+                    cache.size(CachePeekMode.PRIMARY);
+
+                    if (callAsync(cache.<Integer>future(), 0))
+                        return null;
+                }
 
-                if (futs[0] == null && callAsync(new VisorCacheSizeCallable(cache), 0))
-                    return null;
+                if (futs[1] == null) {
+                    cache.clear();
 
-                if (futs[1] == null && callAsync(new VisorCacheClearCallable(cache), 1))
-                    return null;
+                    if (callAsync(cache.<Integer>future(), 1))
+                        return null;
+                }
+                
+                if (futs[2] == null) {
+                    cache.size(CachePeekMode.PRIMARY);
 
-                if (futs[2] == null && callAsync(new VisorCacheSizeCallable(cache), 2))
-                    return null;
+                    if (callAsync(cache.<Integer>future(), 2))
+                        return null;
+                }
             }
 
             assert futs[0].isDone() && futs[1].isDone() && futs[2].isDone();
@@ -141,54 +145,4 @@ private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
             return S.toString(VisorCacheClearJob.class, this);
         }
     }
-
-    /**
-     * Callable to get cache size.
-     */
-    @GridInternal
-    private static class VisorCacheSizeCallable implements IgniteCallable<Integer> {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /** */
-        private final IgniteCache cache;
-
-        /**
-         * @param cache Cache to take size from.
-         */
-        private VisorCacheSizeCallable(IgniteCache cache) {
-            this.cache = cache;
-        }
-
-        /** {@inheritDoc} */
-        @Override public Integer call() throws Exception {
-            return cache.size(CachePeekMode.PRIMARY);
-        }
-    }
-
-    /**
-     * Callable to clear cache.
-     */
-    @GridInternal
-    private static class VisorCacheClearCallable implements IgniteCallable<Integer> {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /** */
-        private final IgniteCache cache;
-
-        /**
-         * @param cache Cache to clear.
-         */
-        private VisorCacheClearCallable(IgniteCache cache) {
-            this.cache = cache;
-        }
-
-        /** {@inheritDoc} */
-        @Override public Integer call() throws Exception {
-            cache.clear();
-
-            return 0;
-        }
-    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index 2539a26c558c9..a64ec6d506d6b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -29,21 +29,26 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.compute.ComputeJobResult;
 import org.apache.ignite.compute.ComputeJobResultPolicy;
 import org.apache.ignite.compute.ComputeTask;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.lang.GridTuple3;
+import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.JobContextResource;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -101,9 +106,16 @@ private static class VisorGatewayJob extends ComputeJobAdapter {
         @IgniteInstanceResource
         protected transient IgniteEx ignite;
 
+        /** Auto-inject job context. */
+        @JobContextResource
+        protected transient ComputeJobContext jobCtx;
+
         /** Arguments count. */
         private final int argsCnt;
 
+        /** Future for spawned task. */
+        private transient IgniteFuture fut;
+
         /**
          * Create job with specified argument.
          *
@@ -284,6 +296,9 @@ private static boolean isBuildInObject(Class cls) {
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
         @Override public Object execute() throws IgniteException {
+            if (fut != null)
+                return fut.get();
+
             String nidsArg = argument(0);
             String taskName = argument(1);
 
@@ -355,8 +370,19 @@ else if (isBuildInObject(argCls))
                 }
             }
 
-            return ignite.compute(ignite.cluster().forNodeIds(nids))
-                .execute(taskName, new VisorTaskArgument<>(nids, jobArgs, false));
+            IgniteCompute comp = ignite.compute(ignite.cluster().forNodeIds(nids)).withAsync();
+            
+            comp.execute(taskName, new VisorTaskArgument<>(nids, jobArgs, false));
+
+            fut = comp.future();
+
+            fut.listen(new CI1<IgniteFuture<Object>>() {
+                @Override public void apply(IgniteFuture<Object> f) {
+                    jobCtx.callcc();
+                }
+            });
+
+            return jobCtx.holdcc();
         }
     }
 }

From 5f5cce4f36e6bc0a3468ba1d80fa5f3d158e927f Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@gridgain.com>
Date: Wed, 1 Mar 2017 22:09:40 +0700
Subject: [PATCH 034/516] Implemented support for enforce join order flag.
 (cherry picked from commit a7f77d4)

---
 .../internal/visor/query/VisorQueryArgV3.java | 51 +++++++++++++++++++
 .../internal/visor/query/VisorQueryJob.java   |  6 +--
 .../resources/META-INF/classnames.properties  |  1 +
 3 files changed, 55 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java
new file mode 100644
index 0000000000000..f32c00a79484b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+/**
+ * Arguments for {@link VisorQueryTask}.
+ */
+public class VisorQueryArgV3 extends VisorQueryArgV2 {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Enforce join order flag. */
+    private final boolean enforceJoinOrder;
+
+    /**
+     * @param cacheName Cache name for query.
+     * @param qryTxt Query text.
+     * @param distributedJoins If {@code true} then distributed joins enabled.
+     * @param enforceJoinOrder If {@code true} then enforce join order.
+     * @param loc Flag whether to execute query locally.
+     * @param pageSize Result batch size.
+     */
+    public VisorQueryArgV3(String cacheName, String qryTxt,
+        boolean distributedJoins, boolean enforceJoinOrder, boolean loc, int pageSize) {
+        super(cacheName, qryTxt, distributedJoins, loc, pageSize);
+
+        this.enforceJoinOrder = enforceJoinOrder;
+    }
+
+    /**
+     * @return Enforce join order flag.
+     */
+    public boolean enforceJoinOrder() {
+        return enforceJoinOrder;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
index c66b2dda8d22c..1ac90ad6a3491 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
@@ -131,9 +131,8 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 if (scanWithFilter) {
                     boolean caseSensitive = qryTxt.startsWith(SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE);
 
-                    String ptrn = caseSensitive
-                        ? qryTxt.substring(SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE.length())
-                        : qryTxt.substring(SCAN_CACHE_WITH_FILTER.length());
+                    String ptrn = qryTxt.substring(
+                        caseSensitive ? SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE.length() : SCAN_CACHE_WITH_FILTER.length());
 
                     filter = new VisorQueryScanSubstringFilter(caseSensitive, ptrn);
                 }
@@ -162,6 +161,7 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 qry.setPageSize(arg.pageSize());
                 qry.setLocal(arg.local());
                 qry.setDistributedJoins(arg instanceof VisorQueryArgV2 && ((VisorQueryArgV2)arg).distributedJoins());
+                qry.setEnforceJoinOrder(arg instanceof VisorQueryArgV3 && ((VisorQueryArgV3)arg).enforceJoinOrder());
 
                 long start = U.currentTimeMillis();
 
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index df605dafb2b34..7bebe24f8a51b 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -1819,6 +1819,7 @@ org.apache.ignite.internal.visor.node.VisorSpisConfiguration
 org.apache.ignite.internal.visor.node.VisorTransactionConfiguration
 org.apache.ignite.internal.visor.query.VisorQueryArg
 org.apache.ignite.internal.visor.query.VisorQueryArgV2
+org.apache.ignite.internal.visor.query.VisorQueryArgV3
 org.apache.ignite.internal.visor.query.VisorQueryCleanupTask
 org.apache.ignite.internal.visor.query.VisorQueryCleanupTask$VisorQueryCleanupJob
 org.apache.ignite.internal.visor.query.VisorQueryField

From 7292a5306c90d5c48c520bf9ffad388769250488 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Mon, 20 Feb 2017 18:23:33 +0700
Subject: [PATCH 035/516] IGNITE-4717 Fixed hangs in VisorCacheClearTask.
 (cherry picked from commit 76f3060)

---
 .../visor/cache/VisorCacheClearTask.java      | 88 +++++--------------
 .../visor/compute/VisorGatewayTask.java       | 30 ++++++-
 2 files changed, 49 insertions(+), 69 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
index 1f1a6fb106bd3..0c8476fa91b10 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.internal.visor.cache;
 
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.internal.processors.task.GridInternal;
@@ -26,7 +25,6 @@
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.resources.JobContextResource;
@@ -90,17 +88,11 @@ private VisorCacheClearJob(String cacheName, boolean debug) {
         }
 
         /**
-         * @param subJob Sub job to execute asynchronously.
+         * @param fut Future for asynchronous cache operation.
          * @param idx Index.
          * @return {@code true} If subJob was not completed and this job should be suspended.
          */
-        private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
-            IgniteCompute compute = ignite.compute(ignite.cluster().forCacheNodes(cacheName)).withAsync();
-
-            compute.call(subJob);
-
-            IgniteFuture<Integer> fut = compute.future();
-
+        private boolean callAsync(IgniteFuture<Integer> fut, int idx) {
             futs[idx] = fut;
 
             if (fut.isDone())
@@ -119,16 +111,28 @@ private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
                 futs = new IgniteFuture[3];
 
             if (futs[0] == null || futs[1] == null || futs[2] == null) {
-                IgniteCache cache = ignite.cache(cacheName);
+                IgniteCache cache = ignite.cache(cacheName).withAsync();
+
+                if (futs[0] == null) {
+                    cache.size(CachePeekMode.PRIMARY);
+
+                    if (callAsync(cache.<Integer>future(), 0))
+                        return null;
+                }
 
-                if (futs[0] == null && callAsync(new VisorCacheSizeCallable(cache), 0))
-                    return null;
+                if (futs[1] == null) {
+                    cache.clear();
 
-                if (futs[1] == null && callAsync(new VisorCacheClearCallable(cache), 1))
-                    return null;
+                    if (callAsync(cache.<Integer>future(), 1))
+                        return null;
+                }
+                
+                if (futs[2] == null) {
+                    cache.size(CachePeekMode.PRIMARY);
 
-                if (futs[2] == null && callAsync(new VisorCacheSizeCallable(cache), 2))
-                    return null;
+                    if (callAsync(cache.<Integer>future(), 2))
+                        return null;
+                }
             }
 
             assert futs[0].isDone() && futs[1].isDone() && futs[2].isDone();
@@ -141,54 +145,4 @@ private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
             return S.toString(VisorCacheClearJob.class, this);
         }
     }
-
-    /**
-     * Callable to get cache size.
-     */
-    @GridInternal
-    private static class VisorCacheSizeCallable implements IgniteCallable<Integer> {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /** */
-        private final IgniteCache cache;
-
-        /**
-         * @param cache Cache to take size from.
-         */
-        private VisorCacheSizeCallable(IgniteCache cache) {
-            this.cache = cache;
-        }
-
-        /** {@inheritDoc} */
-        @Override public Integer call() throws Exception {
-            return cache.size(CachePeekMode.PRIMARY);
-        }
-    }
-
-    /**
-     * Callable to clear cache.
-     */
-    @GridInternal
-    private static class VisorCacheClearCallable implements IgniteCallable<Integer> {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /** */
-        private final IgniteCache cache;
-
-        /**
-         * @param cache Cache to clear.
-         */
-        private VisorCacheClearCallable(IgniteCache cache) {
-            this.cache = cache;
-        }
-
-        /** {@inheritDoc} */
-        @Override public Integer call() throws Exception {
-            cache.clear();
-
-            return 0;
-        }
-    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index 2539a26c558c9..a64ec6d506d6b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -29,21 +29,26 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.compute.ComputeJobResult;
 import org.apache.ignite.compute.ComputeJobResultPolicy;
 import org.apache.ignite.compute.ComputeTask;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.lang.GridTuple3;
+import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.JobContextResource;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -101,9 +106,16 @@ private static class VisorGatewayJob extends ComputeJobAdapter {
         @IgniteInstanceResource
         protected transient IgniteEx ignite;
 
+        /** Auto-inject job context. */
+        @JobContextResource
+        protected transient ComputeJobContext jobCtx;
+
         /** Arguments count. */
         private final int argsCnt;
 
+        /** Future for spawned task. */
+        private transient IgniteFuture fut;
+
         /**
          * Create job with specified argument.
          *
@@ -284,6 +296,9 @@ private static boolean isBuildInObject(Class cls) {
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
         @Override public Object execute() throws IgniteException {
+            if (fut != null)
+                return fut.get();
+
             String nidsArg = argument(0);
             String taskName = argument(1);
 
@@ -355,8 +370,19 @@ else if (isBuildInObject(argCls))
                 }
             }
 
-            return ignite.compute(ignite.cluster().forNodeIds(nids))
-                .execute(taskName, new VisorTaskArgument<>(nids, jobArgs, false));
+            IgniteCompute comp = ignite.compute(ignite.cluster().forNodeIds(nids)).withAsync();
+            
+            comp.execute(taskName, new VisorTaskArgument<>(nids, jobArgs, false));
+
+            fut = comp.future();
+
+            fut.listen(new CI1<IgniteFuture<Object>>() {
+                @Override public void apply(IgniteFuture<Object> f) {
+                    jobCtx.callcc();
+                }
+            });
+
+            return jobCtx.holdcc();
         }
     }
 }

From 620235f3ca0fa32c1f4a4dc1a5e0bd5545f6e07e Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Thu, 2 Mar 2017 11:05:19 +0700
Subject: [PATCH 036/516] Minor cleanup.

---
 .../ignite/internal/visor/cache/VisorCacheTypeMetadata.java | 6 ++++++
 .../ignite/internal/visor/node/VisorIgfsConfiguration.java  | 1 -
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
index f17e5889663fc..c87ad05d824e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.cache.store.jdbc.JdbcTypeField;
 import org.apache.ignite.internal.LessNamingBean;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 
@@ -372,4 +373,9 @@ public Collection<String> txtFlds() {
     public Map<String, LinkedHashMap<String, IgniteBiTuple<String, Boolean>>> grps() {
         return grps;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorCacheTypeMetadata.class, this);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index 9f7652b7d5e3b..cb10c1c3bdc38 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -25,7 +25,6 @@
 import org.apache.ignite.configuration.FileSystemConfiguration;
 import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration;
 import org.apache.ignite.igfs.IgfsMode;
-import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
 import org.apache.ignite.internal.LessNamingBean;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;

From 840ab997436dfad8cdbb4ad182de24bceec37962 Mon Sep 17 00:00:00 2001
From: AKuznetsov <akuznetsov@gridgain.com>
Date: Tue, 14 Feb 2017 20:54:31 +0700
Subject: [PATCH 037/516] IGNITE-4436 API for collecting list of running
 queries and cancel them. (cherry picked from commit 4923734)

---
 .../processors/query/GridQueryIndexing.java   |  17 +-
 .../processors/query/GridQueryProcessor.java  |  32 ++-
 .../query/GridRunningQueryInfo.java           | 132 +++++++++++
 .../internal/visor/VisorMultiNodeTask.java    |   2 +-
 .../visor/query/VisorCancelQueriesTask.java   |  72 ++++++
 .../query/VisorCollectRunningQueriesTask.java |  96 ++++++++
 .../visor/query/VisorRunningQuery.java        | 132 +++++++++++
 .../cache/query/GridCacheTwoStepQuery.java    |  18 +-
 .../processors/query/h2/IgniteH2Indexing.java |  86 +++++++-
 .../query/h2/sql/GridSqlQuerySplitter.java    |   4 +-
 .../h2/twostep/GridReduceQueryExecutor.java   |  60 ++++-
 .../cache/CacheSqlQueryValueCopySelfTest.java | 208 +++++++++++++++++-
 .../GridCacheCrossCacheQuerySelfTest.java     |   2 +-
 .../h2/GridIndexingSpiAbstractSelfTest.java   |   7 +
 14 files changed, 826 insertions(+), 42 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index ef39d96f4d5ec..ca047244a97f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -240,8 +240,23 @@ public void store(@Nullable String spaceName, GridQueryTypeDescriptor type, Cach
      */
     public PreparedStatement prepareNativeStatement(String schema, String sql) throws SQLException;
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> runningQueries(long duration);
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries ID's to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries);
+
     /**
      * Cancels all executing queries.
      */
     public void cancelAllQueries();
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index f8919a105b601..10bf75a01113d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -44,7 +44,6 @@
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryField;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryObjectBuilder;
@@ -119,7 +118,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
     private static final int QRY_DETAIL_METRICS_EVICTION_FREQ = 3_000;
 
     /** */
-    private static Set<Class<?>> SQL_TYPES = new HashSet<>(F.<Class<?>>asList(
+    private static final Set<Class<?>> SQL_TYPES = new HashSet<>(F.<Class<?>>asList(
         Integer.class,
         Boolean.class,
         Byte.class,
@@ -919,6 +918,29 @@ public <K, V> Iterator<Cache.Entry<K, V>> queryLocal(
         }
     }
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+        if (moduleEnabled())
+            return idx.runningQueries(duration);
+
+        return Collections.emptyList();
+    }
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries ID's to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries) {
+        if (moduleEnabled())
+            idx.cancelQueries(queries);
+    }
+
     /**
      * @param sqlQry Sql query.
      * @param params Params.
@@ -2722,7 +2744,7 @@ private interface PropertyAccessor {
     }
 
     /** Accessor that deals with fields. */
-    private final static class FieldAccessor implements PropertyAccessor {
+    private static final class FieldAccessor implements PropertyAccessor {
         /** Field to access. */
         private final Field fld;
 
@@ -2765,7 +2787,7 @@ private FieldAccessor(Field fld) {
     }
 
     /** Getter and setter methods based accessor. */
-    private final static class MethodsAccessor implements PropertyAccessor {
+    private static final class MethodsAccessor implements PropertyAccessor {
         /** */
         private final Method getter;
 
@@ -2823,7 +2845,7 @@ private MethodsAccessor(Method getter, Method setter, String propName) {
     }
 
     /** Accessor with getter only. */
-    private final static class ReadOnlyMethodsAccessor implements PropertyAccessor {
+    private static final class ReadOnlyMethodsAccessor implements PropertyAccessor {
         /** */
         private final Method getter;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
new file mode 100644
index 0000000000000..d77c8c01a8ede
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridRunningQueryInfo.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
+
+/**
+ * Query descriptor.
+ */
+public class GridRunningQueryInfo {
+    /** */
+    private final long id;
+
+    /** */
+    private final String qry;
+
+    /** Query type. */
+    private final GridCacheQueryType qryType;
+
+    /** */
+    private final String cache;
+
+    /** */
+    private final long startTime;
+
+    /** */
+    private final GridQueryCancel cancel;
+
+    /** */
+    private final boolean loc;
+
+    /**
+     * @param id Query ID.
+     * @param qry Query text.
+     * @param qryType Query type.
+     * @param cache Cache where query was executed.
+     * @param startTime Query start time.
+     * @param cancel Query cancel.
+     * @param loc Local query flag.
+     */
+    public GridRunningQueryInfo(Long id, String qry, GridCacheQueryType qryType, String cache, long startTime,
+        GridQueryCancel cancel, boolean loc) {
+        this.id = id;
+        this.qry = qry;
+        this.qryType = qryType;
+        this.cache = cache;
+        this.startTime = startTime;
+        this.cancel = cancel;
+        this.loc = loc;
+    }
+
+    /**
+     * @return Query ID.
+     */
+    public Long id() {
+        return id;
+    }
+
+    /**
+     * @return Query text.
+     */
+    public String query() {
+        return qry;
+    }
+
+    /**
+     * @return Query type.
+     */
+    public GridCacheQueryType queryType() {
+        return qryType;
+    }
+
+    /**
+     * @return Cache where query was executed.
+     */
+    public String cache() {
+        return cache;
+    }
+
+    /**
+     * @return Query start time.
+     */
+    public long startTime() {
+        return startTime;
+    }
+
+    /**
+     * @param curTime Current time.
+     * @param duration Duration of long query.
+     * @return {@code true} if this query should be considered as long running query.
+     */
+    public boolean longQuery(long curTime, long duration) {
+        return curTime - startTime > duration;
+    }
+
+    /**
+     * Cancel query.
+     */
+    public void cancel() {
+        if (cancel != null)
+            cancel.cancel();
+    }
+
+    /**
+     * @return {@code true} if query can be cancelled.
+     */
+    public boolean cancelable() {
+        return cancel != null;
+    }
+
+    /**
+     * @return {@code true} if query is local.
+     */
+    public boolean local() {
+        return loc;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
index 57f134698ea56..ece1a17953f39 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
@@ -130,4 +130,4 @@ protected Map<? extends ComputeJob, ClusterNode> map0(List<ClusterNode> subgrid,
                 logFinish(ignite.log(), getClass(), start);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
new file mode 100644
index 0000000000000..a6f2d821f15a0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCancelQueriesTask.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task to cancel queries.
+ */
+@GridInternal
+public class VisorCancelQueriesTask extends VisorOneNodeTask<Collection<Long>, Void> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCancelQueriesJob job(Collection<Long> arg) {
+        return new VisorCancelQueriesJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected Void reduce0(List<ComputeJobResult> results) throws IgniteException {
+        return null;
+    }
+
+    /**
+     * Job to cancel queries on node.
+     */
+    private static class VisorCancelQueriesJob extends VisorJob<Collection<Long>, Void> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with specified argument.
+         *
+         * @param arg Job argument.
+         * @param debug Flag indicating whether debug information should be printed into node log.
+         */
+        protected VisorCancelQueriesJob(@Nullable Collection<Long> arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Void run(@Nullable Collection<Long> queries) throws IgniteException {
+            ignite.context().query().cancelQueries(queries);
+
+            return null;
+        }
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
new file mode 100644
index 0000000000000..2b40e61e74f50
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorCollectRunningQueriesTask.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task to collect currently running queries.
+ */
+@GridInternal
+public class VisorCollectRunningQueriesTask extends VisorMultiNodeTask<Long, Map<UUID, Collection<VisorRunningQuery>>, Collection<VisorRunningQuery>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCollectRunningQueriesJob job(Long arg) {
+        return new VisorCollectRunningQueriesJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected Map<UUID, Collection<VisorRunningQuery>> reduce0(List<ComputeJobResult> results) throws IgniteException {
+        Map<UUID, Collection<VisorRunningQuery>> map = new HashMap<>();
+
+        for (ComputeJobResult res : results)
+            if (res.getException() == null) {
+                Collection<VisorRunningQuery> queries = res.getData();
+
+                map.put(res.getNode().id(), queries);
+            }
+
+        return map;
+    }
+
+    /**
+     * Job to collect currently running queries from node.
+     */
+    private static class VisorCollectRunningQueriesJob extends VisorJob<Long, Collection<VisorRunningQuery>> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with specified argument.
+         *
+         * @param arg Job argument.
+         * @param debug Flag indicating whether debug information should be printed into node log.
+         */
+        protected VisorCollectRunningQueriesJob(@Nullable Long arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Collection<VisorRunningQuery> run(@Nullable Long duration) throws IgniteException {
+            Collection<GridRunningQueryInfo> queries = ignite.context().query()
+                .runningQueries(duration != null ? duration : 0);
+
+            Collection<VisorRunningQuery> res = new ArrayList<>(queries.size());
+
+            long curTime = U.currentTimeMillis();
+
+            for (GridRunningQueryInfo qry : queries)
+                res.add(new VisorRunningQuery(qry.id(), qry.query(), qry.queryType(), qry.cache(),
+                    qry.startTime(), curTime - qry.startTime(),
+                    qry.cancelable(), qry.local()));
+
+            return res;
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java
new file mode 100644
index 0000000000000..fc6bc7a9222d4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorRunningQuery.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
+
+/**
+ * Descriptor of running query.
+ */
+public class VisorRunningQuery implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private long id;
+
+    /** Query text. */
+    private String qry;
+
+    /** Query type. */
+    private GridCacheQueryType qryType;
+
+    /** Cache name for query. */
+    private String cache;
+
+    /** */
+    private long startTime;
+
+    /** */
+    private long duration;
+
+    /** */
+    private boolean cancellable;
+
+    /** */
+    private boolean loc;
+
+    /**
+     * @param id Query ID.
+     * @param qry Query text.
+     * @param qryType Query type.
+     * @param cache Cache where query was executed.
+     * @param startTime Query start time.
+     * @param duration Query current duration.
+     * @param cancellable {@code true} if query can be canceled.
+     * @param loc {@code true} if query is local.
+     */
+    public VisorRunningQuery(long id, String qry, GridCacheQueryType qryType, String cache,
+        long startTime, long duration,
+        boolean cancellable, boolean loc) {
+        this.id = id;
+        this.qry = qry;
+        this.qryType = qryType;
+        this.cache = cache;
+        this.startTime = startTime;
+        this.duration = duration;
+        this.cancellable = cancellable;
+        this.loc = loc;
+    }
+
+    /**
+     * @return Query ID.
+     */
+    public long getId() {
+        return id;
+    }
+
+    /**
+     * @return Query txt.
+     */
+    public String getQuery() {
+        return qry;
+    }
+
+    /**
+     * @return Query type.
+     */
+    public GridCacheQueryType getQueryType() {
+        return qryType;
+    }
+
+    /**
+     * @return Cache name.
+     */
+    public String getCache() {
+        return cache;
+    }
+
+    /**
+     * @return Query start time.
+     */
+    public long getStartTime() {
+        return startTime;
+    }
+
+    /**
+     * @return Query duration.
+     */
+    public long getDuration() {
+        return duration;
+    }
+
+    /**
+     * @return {@code true} if query can be cancelled.
+     */
+    public boolean isCancelable() {
+        return cancellable;
+    }
+
+    /**
+     * @return {@code true} if query is local.
+     */
+    public boolean isLocal() {
+        return loc;
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
index 8dcba2f2f34dc..f53936fc374a1 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
@@ -45,6 +45,9 @@ public class GridCacheTwoStepQuery {
     /** */
     private boolean explain;
 
+    /** */
+    private String originalSql;
+
     /** */
     private Collection<String> spaces;
 
@@ -67,10 +70,12 @@ public class GridCacheTwoStepQuery {
     private List<Integer> extraCaches;
 
     /**
+     * @param originalSql Original query SQL.
      * @param schemas Schema names in query.
      * @param tbls Tables in query.
      */
-    public GridCacheTwoStepQuery(Set<String> schemas, Set<String> tbls) {
+    public GridCacheTwoStepQuery(String originalSql, Set<String> schemas, Set<String> tbls) {
+        this.originalSql = originalSql;
         this.schemas = schemas;
         this.tbls = tbls;
     }
@@ -195,6 +200,13 @@ public void extraCaches(List<Integer> extraCaches) {
         this.extraCaches = extraCaches;
     }
 
+    /**
+     * @return Original query SQL.
+     */
+    public String originalSql() {
+        return originalSql;
+    }
+
     /**
      * @return Spaces.
      */
@@ -223,7 +235,7 @@ public Set<String> schemas() {
     public GridCacheTwoStepQuery copy(Object[] args) {
         assert !explain;
 
-        GridCacheTwoStepQuery cp = new GridCacheTwoStepQuery(schemas, tbls);
+        GridCacheTwoStepQuery cp = new GridCacheTwoStepQuery(originalSql, schemas, tbls);
 
         cp.caches = caches;
         cp.extraCaches = extraCaches;
@@ -250,4 +262,4 @@ public Set<String> tables() {
     @Override public String toString() {
         return S.toString(GridCacheTwoStepQuery.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index e375867807d0c..8c5c2a364cabf 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -52,6 +52,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import javax.cache.Cache;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
@@ -81,6 +82,7 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
 import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
@@ -176,7 +178,11 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_CLEANUP_PERIOD;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_THREAD_USAGE_TIMEOUT;
+import static org.apache.ignite.IgniteSystemProperties.getInteger;
 import static org.apache.ignite.IgniteSystemProperties.getString;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.TEXT;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.FULLTEXT;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.GEO_SPATIAL;
 import static org.apache.ignite.internal.processors.query.GridQueryIndexType.SORTED;
@@ -283,9 +289,15 @@ public class IgniteH2Indexing implements GridQueryIndexing {
     /** space name -> schema name */
     private final Map<String, String> space2schema = new ConcurrentHashMap8<>();
 
+    /** */
+    private AtomicLong qryIdGen;
+
     /** */
     private GridSpinBusyLock busyLock;
 
+    /** */
+    private final ConcurrentMap<Long, GridRunningQueryInfo> runs = new ConcurrentHashMap8<>();
+
     /** */
     private final ThreadLocal<ConnectionWrapper> connCache = new ThreadLocal<ConnectionWrapper>() {
         @Nullable @Override public ConnectionWrapper get() {
@@ -771,8 +783,19 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
         IndexingQueryFilter filters) throws IgniteCheckedException {
         TableDescriptor tbl = tableDescriptor(spaceName, type);
 
-        if (tbl != null && tbl.luceneIdx != null)
-            return tbl.luceneIdx.query(qry, filters);
+        if (tbl != null && tbl.luceneIdx != null) {
+            GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, TEXT, spaceName,
+                U.currentTimeMillis(), null, true);
+
+            try {
+                runs.put(run.id(), run);
+
+                return tbl.luceneIdx.query(qry, filters);
+            }
+            finally {
+                runs.remove(run.id());
+            }
+        }
 
         return new GridEmptyCloseableIterator<>();
     }
@@ -830,6 +853,11 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
 
                 GridH2QueryContext.set(ctx);
 
+                GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL_FIELDS,
+                    spaceName, U.currentTimeMillis(), cancel, true);
+
+                runs.putIfAbsent(run.id(), run);
+
                 try {
                     ResultSet rs = executeSqlQueryWithTimer(spaceName, stmt, conn, qry, params, timeout, cancel);
 
@@ -837,6 +865,8 @@ private void removeTable(TableDescriptor tbl) throws IgniteCheckedException {
                 }
                 finally {
                     GridH2QueryContext.clearThreadLocal();
+
+                    runs.remove(run.id());
                 }
             }
         };
@@ -1086,6 +1116,11 @@ public void setupConnection(Connection conn, boolean distributedJoins, boolean e
 
         GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter).distributedJoins(false));
 
+        GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL, spaceName,
+            U.currentTimeMillis(), null, true);
+
+        runs.put(run.id(), run);
+
         try {
             ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, null);
 
@@ -1093,6 +1128,8 @@ public void setupConnection(Connection conn, boolean distributedJoins, boolean e
         }
         finally {
             GridH2QueryContext.clearThreadLocal();
+
+            runs.remove(run.id());
         }
     }
 
@@ -1232,7 +1269,7 @@ public static Session session(Connection c) {
                             try {
                                 ctx.cache().createMissingCaches();
                             }
-                            catch (IgniteCheckedException e1) {
+                            catch (IgniteCheckedException ignored) {
                                 throw new CacheException("Failed to create missing caches.", e);
                             }
 
@@ -1737,6 +1774,8 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
 
         this.busyLock = busyLock;
 
+        qryIdGen = new AtomicLong();
+
         if (SysProperties.serializeJavaObject) {
             U.warn(log, "Serialization of Java objects in H2 was enabled.");
 
@@ -1787,7 +1826,7 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
             marshaller = ctx.config().getMarshaller();
 
             mapQryExec = new GridMapQueryExecutor(busyLock);
-            rdcQryExec = new GridReduceQueryExecutor(busyLock);
+            rdcQryExec = new GridReduceQueryExecutor(qryIdGen, busyLock);
 
             mapQryExec.start(ctx, this);
             rdcQryExec.start(ctx, this);
@@ -2241,6 +2280,37 @@ private static List<IndexColumn> treeIndexColumns(List<IndexColumn> cols, IndexC
         return cols;
     }
 
+
+    /** {@inheritDoc} */
+    @Override public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+        Collection<GridRunningQueryInfo> res = new ArrayList<>();
+
+        res.addAll(runs.values());
+        res.addAll(rdcQryExec.longRunningQueries(duration));
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancelQueries(Collection<Long> queries) {
+        if (!F.isEmpty(queries)) {
+            for (Long qryId : queries) {
+                GridRunningQueryInfo run = runs.get(qryId);
+
+                if (run != null)
+                    run.cancel();
+            }
+
+            rdcQryExec.cancelQueries(queries);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cancelAllQueries() {
+        for (Connection conn : conns)
+            U.close(conn, log);
+    }
+
     /**
      * Wrapper to store connection and flag is schema set or not.
      */
@@ -3151,10 +3221,4 @@ private void updateLastUsage() {
             lastUsage = U.currentTimeMillis();
         }
     }
-
-    /** {@inheritDoc} */
-    @Override public void cancelAllQueries() {
-        for (Connection conn : conns)
-            U.close(conn, log);
-    }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 09952cfb8248c..e16431506967a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -174,7 +174,7 @@ public static GridCacheTwoStepQuery split(
         qry = collectAllTables(qry, schemas, tbls);
 
         // Build resulting two step query.
-        GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(schemas, tbls);
+        GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(qry.getSQL(), schemas, tbls);
 
         // Map query will be direct reference to the original query AST.
         // Thus all the modifications will be performed on the original AST, so we should be careful when
@@ -958,4 +958,4 @@ private static GridSqlOperation op(GridSqlOperationType type, GridSqlElement lef
     private static GridSqlFunction function(GridSqlFunctionType type) {
         return new GridSqlFunction(type);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 1f00ed271dabe..ee9976c22846a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -62,6 +62,7 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
 import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
@@ -98,6 +99,7 @@
 import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
 import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REDUCE;
 
 /**
@@ -120,7 +122,7 @@ public class GridReduceQueryExecutor {
     private IgniteLogger log;
 
     /** */
-    private final AtomicLong reqIdGen = new AtomicLong();
+    private final AtomicLong qryIdGen;
 
     /** */
     private final ConcurrentMap<Long, QueryRun> runs = new ConcurrentHashMap8<>();
@@ -167,9 +169,11 @@ public class GridReduceQueryExecutor {
     };
 
     /**
+     * @param qryIdGen Query ID generator.
      * @param busyLock Busy lock.
      */
-    public GridReduceQueryExecutor(GridSpinBusyLock busyLock) {
+    public GridReduceQueryExecutor(AtomicLong qryIdGen, GridSpinBusyLock busyLock) {
+        this.qryIdGen = qryIdGen;
         this.busyLock = busyLock;
     }
 
@@ -493,11 +497,13 @@ public Iterator<List<?>> query(
                 }
             }
 
-            final long qryReqId = reqIdGen.incrementAndGet();
+            final long qryReqId = qryIdGen.incrementAndGet();
 
             final String space = cctx.name();
 
-            final QueryRun r = new QueryRun(h2.connectionForSpace(space), qry.mapQueries().size(), qry.pageSize());
+            final QueryRun r = new QueryRun(qryReqId, qry.originalSql(), space,
+                h2.connectionForSpace(space), qry.mapQueries().size(), qry.pageSize(),
+                U.currentTimeMillis(), cancel);
 
             AffinityTopologyVersion topVer = h2.readyTopologyVersion();
 
@@ -1302,10 +1308,46 @@ public void onDisconnected(IgniteFuture<?> reconnectFut) {
             e.getValue().disconnected(err);
     }
 
+    /**
+     * Collect queries that already running more than specified duration.
+     *
+     * @param duration Duration to check.
+     * @return Collection of IDs and statements of long running queries.
+     */
+    public Collection<GridRunningQueryInfo> longRunningQueries(long duration) {
+        Collection<GridRunningQueryInfo> res = new ArrayList<>();
+
+        long curTime = U.currentTimeMillis();
+
+        for (QueryRun run : runs.values()) {
+            if (run.qry.longQuery(curTime, duration))
+                res.add(run.qry);
+        }
+
+        return res;
+    }
+
+    /**
+     * Cancel specified queries.
+     *
+     * @param queries Queries IDs to cancel.
+     */
+    public void cancelQueries(Collection<Long> queries) {
+        for (Long qryId : queries) {
+            QueryRun run = runs.get(qryId);
+
+            if (run != null)
+                run.qry.cancel();
+        }
+    }
+
     /**
      * Query run.
      */
     private static class QueryRun {
+        /** */
+        private final GridRunningQueryInfo qry;
+
         /** */
         private final List<GridMergeIndex> idxs;
 
@@ -1322,11 +1364,17 @@ private static class QueryRun {
         private final AtomicReference<Object> state = new AtomicReference<>();
 
         /**
+         * @param id Query ID.
+         * @param qry Query text.
+         * @param cache Cache where query was executed.
          * @param conn Connection.
          * @param idxsCnt Number of indexes.
          * @param pageSize Page size.
+         * @param startTime Start time.
+         * @param cancel Query cancel handler.
          */
-        private QueryRun(Connection conn, int idxsCnt, int pageSize) {
+        private QueryRun(Long id, String qry, String cache, Connection conn, int idxsCnt, int pageSize, long startTime, GridQueryCancel cancel) {
+            this.qry = new GridRunningQueryInfo(id, qry, SQL_FIELDS, cache, startTime, cancel, false);
             this.conn = (JdbcConnection)conn;
             this.idxs = new ArrayList<>(idxsCnt);
             this.pageSize = pageSize > 0 ? pageSize : GridCacheTwoStepQuery.DFLT_PAGE_SIZE;
@@ -1384,4 +1432,4 @@ private ExplicitPartitionsSpecializer(Map<ClusterNode,IntArray> partsMap) {
             return copy(msg, n, partsMap);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
index e47e893530384..66e7e4abf9cdb 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheSqlQueryValueCopySelfTest.java
@@ -17,15 +17,23 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.Query;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cache.query.annotations.QuerySqlFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -54,6 +62,7 @@ public class CacheSqlQueryValueCopySelfTest extends GridCommonAbstractTest {
 
         cc.setCopyOnRead(true);
         cc.setIndexedTypes(Integer.class, Value.class);
+        cc.setSqlFunctionClasses(TestSQLFunctions.class);
 
         cfg.setCacheConfiguration(cc);
 
@@ -72,7 +81,7 @@ public class CacheSqlQueryValueCopySelfTest extends GridCommonAbstractTest {
         IgniteCache<Integer, Value> cache = grid(0).cache(null);
 
         for (int i = 0; i < KEYS; i++)
-            cache.put(i, new Value("before"));
+            cache.put(i, new Value(i, "before-" + i));
     }
 
     /** {@inheritDoc} */
@@ -195,17 +204,148 @@ public void testLocalSqlFieldsQuery() {
         check(cache);
     }
 
-    /** */
-    private static class Value {
-        /** */
-        private String str;
+    /**
+     * Run specified query in separate thread.
+     *
+     * @param qry Query to execute.
+     */
+    private IgniteInternalFuture<?> runQueryAsync(final Query<?> qry) throws Exception {
+        return multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    log.info(">>> Query started");
+
+                    grid(0).cache(null).query(qry).getAll();
+
+                    log.info(">>> Query finished");
+                }
+                catch (Throwable e) {
+                    e.printStackTrace();
+                }
+            }
+        }, 1, "run-query");
+    }
 
-        /**
-         * @param str String.
-         */
-        public Value(String str) {
-            this.str = str;
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRunningSqlFieldsQuery() throws Exception {
+        IgniteInternalFuture<?> fut = runQueryAsync(new SqlFieldsQuery("select _val, sleep(1000) from Value limit 3"));
+
+        Thread.sleep(500);
+
+        GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+
+        SqlFieldsQuery qry = new SqlFieldsQuery("select _val, sleep(1000) from Value limit 3");
+        qry.setLocal(true);
+
+        fut = runQueryAsync(qry);
+
+        Thread.sleep(500);
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+    }
+
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRunningSqlQuery() throws Exception {
+        IgniteInternalFuture<?> fut = runQueryAsync(new SqlQuery<Integer, Value>(Value.class, "id > sleep(100)"));
+
+        Thread.sleep(500);
+
+        GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+
+        SqlQuery<Integer, Value> qry = new SqlQuery<>(Value.class, "id > sleep(100)");
+        qry.setLocal(true);
+
+        fut = runQueryAsync(qry);
+
+        Thread.sleep(500);
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        fut.get();
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
+    }
+
+    /**
+     * Test collecting info about running.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCancelingSqlFieldsQuery() throws Exception {
+        runQueryAsync(new SqlFieldsQuery("select * from (select _val, sleep(100) from Value limit 50)"));
+
+        Thread.sleep(500);
+
+        final GridQueryProcessor qryProc = grid(0).context().query();
+
+        Collection<GridRunningQueryInfo> queries = qryProc.runningQueries(0);
+
+        assertEquals(1, queries.size());
+
+        final Collection<GridRunningQueryInfo> finalQueries = queries;
+
+        for (GridRunningQueryInfo query : finalQueries)
+            qryProc.cancelQueries(Collections.singleton(query.id()));
+
+        int n = 100;
+
+        // Give cluster some time to cancel query and cleanup resources.
+        while (n > 0) {
+            Thread.sleep(100);
+
+            queries = qryProc.runningQueries(0);
+
+            if (queries.isEmpty())
+                break;
+
+            log.info(">>>> Wait for cancel: " + n);
+
+            n--;
         }
+
+        queries = qryProc.runningQueries(0);
+
+        assertEquals(0, queries.size());
     }
 
     /**
@@ -218,9 +358,53 @@ private void check(IgniteCache<Integer, Value> cache) {
         for (Cache.Entry<Integer, Value> entry : cache) {
             cnt++;
 
-            assertEquals("before", entry.getValue().str);
+            assertEquals("before-" + entry.getKey(), entry.getValue().str);
         }
 
         assertEquals(KEYS, cnt);
     }
-}
\ No newline at end of file
+
+    /** */
+    private static class Value {
+        /** */
+        @QuerySqlField
+        private int id;
+
+        /** */
+        @QuerySqlField
+        private String str;
+
+        /**
+         * @param id ID.
+         * @param str String.
+         */
+        public Value(int id, String str) {
+            this.id = id;
+            this.str = str;
+        }
+    }
+
+    /**
+     * Utility class with custom SQL functions.
+     */
+    public static class TestSQLFunctions {
+        /**
+         * Sleep function to simulate long running queries.
+         *
+         * @param x Time to sleep.
+         * @return Return specified argument.
+         */
+        @QuerySqlFunction
+        public static long sleep(long x) {
+            if (x >= 0)
+                try {
+                    Thread.sleep(x);
+                }
+                catch (InterruptedException ignored) {
+                    // No-op.
+                }
+
+            return x;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
index 1f10593e61e0c..01fefa3ac95e3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
@@ -477,4 +477,4 @@ public int getStoreId() {
             return storeId;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
index 17151d8730287..09b27d2ca3f77 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
@@ -106,6 +106,9 @@ protected void startIndexing(IgniteH2Indexing spi) throws Exception {
         spi.registerCache(null, cacheCfg("B"));
     }
 
+    /**
+     * @param name Name.
+     */
     private CacheConfiguration cacheCfg(String name) {
         CacheConfiguration<?,?> cfg = new CacheConfiguration<>();
 
@@ -114,6 +117,7 @@ private CacheConfiguration cacheCfg(String name) {
         return cfg;
     }
 
+    /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         idx.stop();
 
@@ -182,6 +186,9 @@ private IgniteH2Indexing getIndexing() {
         return idx;
     }
 
+    /**
+     * @return {@code true} if OFF-HEAP mode should be tested.
+     */
     protected boolean offheap() {
         return false;
     }

From bc9fcf7f6aaeba826df6a35f1a9aacb17a562337 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@gridgain.com>
Date: Wed, 1 Mar 2017 22:09:40 +0700
Subject: [PATCH 038/516] Implemented support for enforce join order flag.
 (cherry picked from commit a7f77d4)

---
 .../internal/visor/query/VisorQueryArgV3.java | 51 +++++++++++++++++++
 .../internal/visor/query/VisorQueryJob.java   |  6 +--
 .../resources/META-INF/classnames.properties  |  1 +
 3 files changed, 55 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java
new file mode 100644
index 0000000000000..f32c00a79484b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryArgV3.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.query;
+
+/**
+ * Arguments for {@link VisorQueryTask}.
+ */
+public class VisorQueryArgV3 extends VisorQueryArgV2 {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Enforce join order flag. */
+    private final boolean enforceJoinOrder;
+
+    /**
+     * @param cacheName Cache name for query.
+     * @param qryTxt Query text.
+     * @param distributedJoins If {@code true} then distributed joins enabled.
+     * @param enforceJoinOrder If {@code true} then enforce join order.
+     * @param loc Flag whether to execute query locally.
+     * @param pageSize Result batch size.
+     */
+    public VisorQueryArgV3(String cacheName, String qryTxt,
+        boolean distributedJoins, boolean enforceJoinOrder, boolean loc, int pageSize) {
+        super(cacheName, qryTxt, distributedJoins, loc, pageSize);
+
+        this.enforceJoinOrder = enforceJoinOrder;
+    }
+
+    /**
+     * @return Enforce join order flag.
+     */
+    public boolean enforceJoinOrder() {
+        return enforceJoinOrder;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
index c66b2dda8d22c..1ac90ad6a3491 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryJob.java
@@ -131,9 +131,8 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 if (scanWithFilter) {
                     boolean caseSensitive = qryTxt.startsWith(SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE);
 
-                    String ptrn = caseSensitive
-                        ? qryTxt.substring(SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE.length())
-                        : qryTxt.substring(SCAN_CACHE_WITH_FILTER.length());
+                    String ptrn = qryTxt.substring(
+                        caseSensitive ? SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE.length() : SCAN_CACHE_WITH_FILTER.length());
 
                     filter = new VisorQueryScanSubstringFilter(caseSensitive, ptrn);
                 }
@@ -162,6 +161,7 @@ private QueryCursor<Cache.Entry<Object, Object>> near(IgniteCache<Object, Object
                 qry.setPageSize(arg.pageSize());
                 qry.setLocal(arg.local());
                 qry.setDistributedJoins(arg instanceof VisorQueryArgV2 && ((VisorQueryArgV2)arg).distributedJoins());
+                qry.setEnforceJoinOrder(arg instanceof VisorQueryArgV3 && ((VisorQueryArgV3)arg).enforceJoinOrder());
 
                 long start = U.currentTimeMillis();
 
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index c0a6407e675ca..8cfed3a182703 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -1822,6 +1822,7 @@ org.apache.ignite.internal.visor.node.VisorSpisConfiguration
 org.apache.ignite.internal.visor.node.VisorTransactionConfiguration
 org.apache.ignite.internal.visor.query.VisorQueryArg
 org.apache.ignite.internal.visor.query.VisorQueryArgV2
+org.apache.ignite.internal.visor.query.VisorQueryArgV3
 org.apache.ignite.internal.visor.query.VisorQueryCleanupTask
 org.apache.ignite.internal.visor.query.VisorQueryCleanupTask$VisorQueryCleanupJob
 org.apache.ignite.internal.visor.query.VisorQueryField

From c2b201ed5bf4bea5dd5aa013f685fdfa02544527 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 2 Mar 2017 10:27:13 +0300
Subject: [PATCH 039/516] IGNITE-3386 - Reentrant lock is lost when owner
 leaves topology

---
 .../main/java/org/apache/ignite/Ignite.java   |  1 +
 .../DataStructuresProcessor.java              |  3 +-
 .../datastructures/GridCacheLockImpl.java     | 17 ++++-
 .../internal/GridCacheRecreateLockTest.java   | 62 +++++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |  2 +
 5 files changed, 82 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java
index 0de08d56584a8..7f8974c20fd9e 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignite.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java
@@ -527,6 +527,7 @@ public IgniteSemaphore semaphore(String name, int cnt, boolean failoverSafe, boo
      *      all threads on other nodes waiting to acquire lock are interrupted.
      * @param fair If {@code True}, fair lock will be created.
      * @param create Boolean flag indicating whether data structure should be created if does not exist.
+     *      Will re-create lock if the node that stored the lock left topology and there are no backups left.
      * @return ReentrantLock for the given name.
      * @throws IgniteException If reentrant lock could not be fetched or created.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
index 1cad22f33e5ef..698efd9d7665d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
@@ -1396,7 +1396,8 @@ public IgniteLock reentrantLock(final String name, final boolean failoverSafe, f
                         name,
                         key,
                         reentrantLockView,
-                        dsCacheCtx);
+                        dsCacheCtx,
+                        create);
 
                     dsMap.put(key, reentrantLock0);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
index 3ab7289237a63..1cf78faa5bd4e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
@@ -105,6 +105,9 @@ public final class GridCacheLockImpl implements GridCacheLockEx, Externalizable
     /** Flag indicating that every operation on this lock should be interrupted. */
     private volatile boolean interruptAll;
 
+    /** Re-create flag. */
+    private volatile boolean reCreate;
+
     /**
      * Empty constructor required by {@link Externalizable}.
      */
@@ -522,7 +525,14 @@ protected boolean compareAndSetGlobalState(final int expVal, final int newVal,
                                 GridCacheLockState val = lockView.get(key);
 
                                 if (val == null)
-                                    throw new IgniteCheckedException("Failed to find reentrant lock with given name: " + name);
+                                    if (reCreate) {
+                                        val = new GridCacheLockState(0, ctx.nodeId(), 0, failoverSafe, fair);
+
+                                        lockView.put(key, val);
+                                    }
+                                    else
+                                        throw new IgniteCheckedException("Failed to find reentrant lock with " +
+                                            "the given name: " + name);
 
                                 final long newThreadID = newThread.getId();
 
@@ -1048,12 +1058,14 @@ protected IgniteConditionObject(String name, ConditionObject object) {
      * @param key Reentrant lock key.
      * @param lockView Reentrant lock projection.
      * @param ctx Cache context.
+     * @param reCreate If {@code true} reentrant lock will be re-created in case it is not in cache.
      */
     @SuppressWarnings("unchecked")
     public GridCacheLockImpl(String name,
         GridCacheInternalKey key,
         IgniteInternalCache<GridCacheInternalKey, GridCacheLockState> lockView,
-        GridCacheContext ctx) {
+        GridCacheContext ctx,
+        boolean reCreate) {
         assert name != null;
         assert key != null;
         assert ctx != null;
@@ -1063,6 +1075,7 @@ public GridCacheLockImpl(String name,
         this.key = key;
         this.lockView = lockView;
         this.ctx = ctx;
+        this.reCreate = reCreate;
 
         log = ctx.logger(getClass());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
new file mode 100644
index 0000000000000..ae4ef87151c1e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
@@ -0,0 +1,62 @@
+package org.apache.ignite.internal;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLock;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+
+/**
+ * Create lock after owner node left topology test
+ */
+@GridCommonTest(group = "Kernal Self")
+public class GridCacheRecreateLockTest extends GridCommonAbstractTest {
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void test() throws Exception {
+        final Ignite ignite = startNodeAndLock("node1");
+
+        new Thread(new Runnable() {
+            @Override public void run() {
+                try {
+                    Thread.sleep(2000);
+                }
+                catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+
+                ignite.close();
+            }
+        }).start();
+
+        startNodeAndLock("node2");
+    }
+
+    private Ignite startNodeAndLock(String name) {
+        try {
+            IgniteConfiguration cfg = new IgniteConfiguration();
+            cfg.setGridName(name);
+
+            Ignite ignite = Ignition.start(cfg);
+
+            IgniteLock lock = ignite.reentrantLock("lock", true, true, true);
+
+            System.out.println("acquiring lock");
+
+            lock.lock();
+
+            System.out.println("acquired lock");
+
+            return ignite;
+        }
+        catch (Exception e) {
+            assertTrue(false);
+        }
+
+        return null;
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 8a501fdb801d6..7c8b6a94c8f0d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.internal.GridAffinityNoCacheSelfTest;
 import org.apache.ignite.internal.GridAffinitySelfTest;
 import org.apache.ignite.internal.GridAlwaysFailoverSpiFailSelfTest;
+import org.apache.ignite.internal.GridCacheRecreateLockTest;
 import org.apache.ignite.internal.GridCancelOnGridStopSelfTest;
 import org.apache.ignite.internal.GridCancelUnusedJobSelfTest;
 import org.apache.ignite.internal.GridCancelledJobsMetricsSelfTest;
@@ -152,6 +153,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TaskNodeRestartTest.class);
         suite.addTestSuite(IgniteRoundRobinErrorAfterClientReconnectTest.class);
         suite.addTestSuite(PublicThreadpoolStarvationTest.class);
+        suite.addTestSuite(GridCacheRecreateLockTest.class);
 
         return suite;
     }

From 3310f10b9d48d64280d0ebf2464ad892f3f31c52 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 2 Mar 2017 10:37:00 +0300
Subject: [PATCH 040/516] IGNITE-3386 - Minor code style changes

---
 .../internal/GridCacheRecreateLockTest.java   | 32 ++++++++-----------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
index ae4ef87151c1e..ae850f7789745 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
@@ -13,11 +13,10 @@
  */
 @GridCommonTest(group = "Kernal Self")
 public class GridCacheRecreateLockTest extends GridCommonAbstractTest {
-
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void test() throws Exception {
+    public void testLockOwnerLeavesGrid() throws Exception {
         final Ignite ignite = startNodeAndLock("node1");
 
         new Thread(new Runnable() {
@@ -36,27 +35,24 @@ public void test() throws Exception {
         startNodeAndLock("node2");
     }
 
-    private Ignite startNodeAndLock(String name) {
-        try {
-            IgniteConfiguration cfg = new IgniteConfiguration();
-            cfg.setGridName(name);
-
-            Ignite ignite = Ignition.start(cfg);
+    /**
+     * @param name Grid name.
+     * @return Started Ignite instance.
+     */
+    private Ignite startNodeAndLock(String name) throws Exception {
+        IgniteConfiguration cfg = new IgniteConfiguration();
+        cfg.setGridName(name);
 
-            IgniteLock lock = ignite.reentrantLock("lock", true, true, true);
+        Ignite ignite = Ignition.start(cfg);
 
-            System.out.println("acquiring lock");
+        IgniteLock lock = ignite.reentrantLock("lock", true, true, true);
 
-            lock.lock();
+        System.out.println("acquiring lock");
 
-            System.out.println("acquired lock");
+        lock.lock();
 
-            return ignite;
-        }
-        catch (Exception e) {
-            assertTrue(false);
-        }
+        System.out.println("acquired lock");
 
-        return null;
+        return ignite;
     }
 }
\ No newline at end of file

From 7ad8e79fa1077291c50f2f535ecccde6baee0321 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Tue, 7 Mar 2017 14:32:28 +0300
Subject: [PATCH 041/516] ignite-4577 Add non-reachable addresses at the end of
 addresses list

---
 .../ignite/internal/util/IgniteUtils.java     | 14 ++++++-----
 .../tcp/TcpCommunicationSpi.java              | 25 +++++++++++++++++++
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 3fa3f7b92eccb..ba118cb118cea 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1810,15 +1810,16 @@ public static synchronized boolean isLocalHostChanged() throws IOException {
 
     /**
      * @param addrs Addresses.
+     * @return List of reachable addresses.
      */
-    public static List<InetAddress> filterReachable(List<InetAddress> addrs) {
+    public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
         final int reachTimeout = 2000;
 
         if (addrs.isEmpty())
             return Collections.emptyList();
 
         if (addrs.size() == 1) {
-            InetAddress addr = addrs.get(0);
+            InetAddress addr = F.first(addrs);
 
             if (reachable(addr, reachTimeout))
                 return Collections.singletonList(addr);
@@ -1834,8 +1835,7 @@ public static List<InetAddress> filterReachable(List<InetAddress> addrs) {
 
         for (final InetAddress addr : addrs) {
             futs.add(executor.submit(new Runnable() {
-                @Override
-                public void run() {
+                @Override public void run() {
                     if (reachable(addr, reachTimeout)) {
                         synchronized (res) {
                             res.add(addr);
@@ -1848,11 +1848,13 @@ public void run() {
         for (Future<?> fut : futs) {
             try {
                 fut.get();
-            } catch (InterruptedException e) {
+            }
+            catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
 
                 throw new IgniteException("Thread has been interrupted.", e);
-            } catch (ExecutionException e) {
+            }
+            catch (ExecutionException e) {
                 throw new IgniteException(e);
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 94b7efe078e84..81454f827e58f 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2334,6 +2334,31 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
         if (isExtAddrsExist)
             addrs.addAll(extAddrs);
 
+        Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
+
+        for (InetSocketAddress addr : addrs)
+            allInetAddrs.add(addr.getAddress());
+
+        List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
+
+        if (reachableInetAddrs.size() < allInetAddrs.size()) {
+            LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
+
+            for (InetSocketAddress addr : addrs) {
+                if (reachableInetAddrs.contains(addr.getAddress()))
+                    addrs0.add(addr);
+            }
+            for (InetSocketAddress addr : addrs) {
+                if (!reachableInetAddrs.contains(addr.getAddress()))
+                    addrs0.add(addr);
+            }
+
+            addrs = addrs0;
+        }
+
+        if (log.isDebugEnabled())
+            log.debug("Addresses to connect for node [rmtNode=" + node.id() + ", addrs=" + addrs.toString() + ']');
+
         boolean conn = false;
         GridCommunicationClient client = null;
         IgniteCheckedException errs = null;

From 8817190e1dd31d869682df0167bb3e82fb597aad Mon Sep 17 00:00:00 2001
From: Sergi Vladykin <sergi.vladykin@gmail.com>
Date: Thu, 9 Mar 2017 23:30:09 +0300
Subject: [PATCH 042/516] ignite-1.9 - SQL related fixes and improvements:  -
 Sorted MERGE index  - EXPLAIN fixes  - Replicated subqueries fixes

Squashed commit of the following:

commit 423c2155c85ed9be8dffb3517b7331b753e1ce5c
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Thu Mar 9 23:21:38 2017 +0300

    ignite-1.9.1 - test fix

commit ff3c1f2967905b0bcac7661014656d1c080fa803
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Thu Mar 9 11:08:34 2017 +0300

    ignite-1.9.0 - replicated subqueries fix

commit bc0801a3c976f5d87cab2c414f76f69dc28b43d7
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Wed Mar 8 16:03:40 2017 +0300

    ignite-1.9.0 - fix for distributed join test

commit f1f1d96c6babaadab9e3ed1fbb3c9740c94d8209
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Wed Mar 8 15:28:44 2017 +0300

    ignite-1.9.0 - fix for distributed join test

commit a8751d535b3e025a804c441204465e94035a5247
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Tue Feb 28 18:46:07 2017 +0300

    ignite-1.9 - splitter fixes

commit 0601ce6e291eb4689d526e922b02fd9e21df5b08
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Sun Feb 26 23:24:14 2017 +0300

    ignite-1.9 - merge index test

commit 4ad048e248157d799a325b3ce9975d4ad8a9fb49
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Sun Feb 26 23:19:49 2017 +0300

    ignite-1.9 - merge index

commit 4ea63d7335000b8f30bfbd1bb907e411cd62a5e8
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Sun Feb 26 22:44:51 2017 +0300

    ignite-1.9 - unsorted index fixed

commit a639bff6f25a8397e49a892f830c9de23c847127
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Sun Feb 26 20:08:26 2017 +0300

    ignite-1.9 - sorted index fixes2

commit ee9d524f5a0d6f1c416345822e8201c327f1e562
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Fri Feb 24 16:00:26 2017 +0300

    ignite-1.9 - sorted index fixes

commit fc42406a9e55851d53d9dfed8e6cf3c8b12af345
Author: Sergi Vladykin <sergi.vladykin@gmail.com>
Date:   Thu Feb 23 16:46:39 2017 +0300

    ignite-1.9 - sorted index
---
 .../cache/query/GridCacheSqlQuery.java        |  82 ++++-
 .../processors/query/h2/IgniteH2Indexing.java |   4 +-
 .../query/h2/opt/GridH2CollocationModel.java  |   6 +-
 .../query/h2/opt/GridH2ScanIndex.java         | 273 ++++++++++++++++
 .../processors/query/h2/opt/GridH2Table.java  | 244 +-------------
 .../processors/query/h2/sql/GridSqlQuery.java |  17 -
 .../query/h2/sql/GridSqlQueryParser.java      |   4 +-
 .../query/h2/sql/GridSqlQuerySplitter.java    |  38 ++-
 .../query/h2/sql/GridSqlSortColumn.java       |  41 +++
 .../h2/twostep/GridMapQueryExecutor.java      |  83 +++--
 .../query/h2/twostep/GridMergeIndex.java      | 300 ++++++++++--------
 .../h2/twostep/GridMergeIndexSorted.java      | 172 +++++++---
 .../h2/twostep/GridMergeIndexUnsorted.java    |  67 +++-
 .../query/h2/twostep/GridMergeTable.java      |  70 +++-
 .../h2/twostep/GridReduceQueryExecutor.java   | 101 ++++--
 .../query/h2/twostep/GridResultPage.java      |  34 +-
 .../h2/twostep/msg/GridH2QueryRequest.java    |  11 +-
 .../IgniteCacheAbstractQuerySelfTest.java     |  10 +-
 .../query/IgniteSqlSplitterSelfTest.java      | 100 +++++-
 .../query/h2/sql/H2CompareBigQueryTest.java   |   4 +-
 .../IgniteCacheQuerySelfTestSuite.java        |   2 +
 .../processors/query/h2/sql/bigQuery.sql      |  34 +-
 22 files changed, 1138 insertions(+), 559 deletions(-)
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2ScanIndex.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
index 18688b766578a..c4bb2056585b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
@@ -19,6 +19,8 @@
 
 import java.nio.ByteBuffer;
 import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.GridDirectTransient;
@@ -74,6 +76,19 @@ public class GridCacheSqlQuery implements Message, GridCacheQueryMarshallable {
     /** Field kept for backward compatibility. */
     private String alias;
 
+    /** Sort columns. */
+    @GridToStringInclude
+    @GridDirectTransient
+    private transient List<?> sort;
+
+    /** If we have partitioned tables in this query. */
+    @GridToStringInclude
+    @GridDirectTransient
+    private transient boolean partitioned;
+
+    /** Single node to execute the query on. */
+    private UUID node;
+
     /**
      * For {@link Message}.
      */
@@ -218,12 +233,18 @@ public GridCacheSqlQuery parameters(Object[] params, int[] paramIdxs) {
                 writer.incrementState();
 
             case 1:
-                if (!writer.writeByteArray("paramsBytes", paramsBytes))
+                if (!writer.writeUuid("node", node))
                     return false;
 
                 writer.incrementState();
 
             case 2:
+                if (!writer.writeByteArray("paramsBytes", paramsBytes))
+                    return false;
+
+                writer.incrementState();
+
+            case 3:
                 if (!writer.writeString("qry", qry))
                     return false;
 
@@ -251,7 +272,7 @@ public GridCacheSqlQuery parameters(Object[] params, int[] paramIdxs) {
                 reader.incrementState();
 
             case 1:
-                paramsBytes = reader.readByteArray("paramsBytes");
+                node = reader.readUuid("node");
 
                 if (!reader.isLastRead())
                     return false;
@@ -259,6 +280,14 @@ public GridCacheSqlQuery parameters(Object[] params, int[] paramIdxs) {
                 reader.incrementState();
 
             case 2:
+                paramsBytes = reader.readByteArray("paramsBytes");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 3:
                 qry = reader.readString("qry");
 
                 if (!reader.isLastRead())
@@ -278,7 +307,7 @@ public GridCacheSqlQuery parameters(Object[] params, int[] paramIdxs) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 3;
+        return 4;
     }
 
     /**
@@ -292,6 +321,8 @@ public GridCacheSqlQuery copy(Object[] args) {
         cp.cols = cols;
         cp.paramIdxs = paramIdxs;
         cp.paramsSize = paramsSize;
+        cp.sort = sort;
+        cp.partitioned = partitioned;
 
         if (F.isEmpty(args))
             cp.params = EMPTY_PARAMS;
@@ -304,4 +335,49 @@ public GridCacheSqlQuery copy(Object[] args) {
 
         return cp;
     }
+
+    /**
+     * @param sort Sort columns.
+     */
+    public void sortColumns(List<?> sort) {
+        this.sort = sort;
+    }
+
+    /**
+     * @return Sort columns.
+     */
+    public List<?> sortColumns() {
+        return sort;
+    }
+
+    /**
+     * @param partitioned If the query contains partitioned tables.
+     */
+    public void partitioned(boolean partitioned) {
+        this.partitioned = partitioned;
+    }
+
+    /**
+     * @return {@code true} If the query contains partitioned tables.
+     */
+    public boolean isPartitioned() {
+        return partitioned;
+    }
+
+    /**
+     * @return Single node to execute the query on or {@code null} if need to execute on all the nodes.
+     */
+    public UUID node() {
+        return node;
+    }
+
+    /**
+     * @param node Single node to execute the query on or {@code null} if need to execute on all the nodes.
+     * @return {@code this}.
+     */
+    public GridCacheSqlQuery node(UUID node) {
+        this.node = node;
+
+        return this;
+    }
 }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index b4bf608b10d8a..8de8dc4bc9bf8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -85,7 +85,6 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
 import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
-import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
 import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
@@ -155,7 +154,6 @@
 import org.h2.server.web.WebServer;
 import org.h2.table.Column;
 import org.h2.table.IndexColumn;
-import org.h2.table.Table;
 import org.h2.tools.Server;
 import org.h2.util.JdbcUtils;
 import org.h2.value.DataType;
@@ -1453,7 +1451,7 @@ public static Session session(Connection c) {
                     }
 
 
-                    Prepared prepared = GridSqlQueryParser.prepared((JdbcPreparedStatement) stmt);
+                    Prepared prepared = GridSqlQueryParser.prepared(stmt);
 
                     if (qry instanceof JdbcSqlFieldsQuery && ((JdbcSqlFieldsQuery) qry).isQuery() != prepared.isQuery())
                         throw new IgniteSQLException("Given statement type does not match that declared by JDBC driver",
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2CollocationModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2CollocationModel.java
index ce11fd59cd4cc..4df355e2529b2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2CollocationModel.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2CollocationModel.java
@@ -300,10 +300,10 @@ private void calculate() {
             assert childFilters == null;
 
             // We are at table instance.
-            GridH2Table tbl = (GridH2Table)filter().getTable();
+            Table tbl = filter().getTable();
 
             // Only partitioned tables will do distributed joins.
-            if (!tbl.isPartitioned()) {
+            if (!(tbl instanceof GridH2Table) || !((GridH2Table)tbl).isPartitioned()) {
                 type = Type.REPLICATED;
                 multiplier = MULTIPLIER_COLLOCATED;
 
@@ -593,7 +593,7 @@ else if (!left.isPartitioned() && !right.isPartitioned())
     private GridH2CollocationModel child(int i, boolean create) {
         GridH2CollocationModel child = children[i];
 
-        if (child == null && create && isChildTableOrView(i, null)) {
+        if (child == null && create) {
             TableFilter f = childFilters[i];
 
             if (f.getTable().isView()) {
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2ScanIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2ScanIndex.java
new file mode 100644
index 0000000000000..3ddd490508f94
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2ScanIndex.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.opt;
+
+import java.util.ArrayList;
+import org.h2.engine.Database;
+import org.h2.engine.DbObject;
+import org.h2.engine.Session;
+import org.h2.index.BaseIndex;
+import org.h2.index.Cursor;
+import org.h2.index.IndexLookupBatch;
+import org.h2.index.IndexType;
+import org.h2.message.DbException;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.schema.Schema;
+import org.h2.table.Column;
+import org.h2.table.IndexColumn;
+import org.h2.table.Table;
+import org.h2.table.TableFilter;
+
+/**
+ * Scan index base class.
+ */
+public abstract class GridH2ScanIndex<D extends BaseIndex> extends BaseIndex {
+    /** */
+    private static final IndexType TYPE = IndexType.createScan(false);
+
+    /** */
+    protected final D delegate;
+
+    /**
+     * @param delegate Delegate.
+     */
+    public GridH2ScanIndex(D delegate) {
+        this.delegate = delegate;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getDiskSpaceUsed() {
+        return 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void add(Session ses, Row row) {
+        delegate.add(ses, row);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canFindNext() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canGetFirstOrLast() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canScan() {
+        return delegate.canScan();
+    }
+
+    /** {@inheritDoc} */
+    @Override public final void close(Session ses) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void commit(int operation, Row row) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public int compareRows(SearchRow rowData, SearchRow compare) {
+        return delegate.compareRows(rowData, compare);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Cursor find(TableFilter filter, SearchRow first, SearchRow last) {
+        return find(filter.getSession(), first, last);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Cursor find(Session ses, SearchRow first, SearchRow last) {
+        return delegate.find(ses, null, null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Cursor findFirstOrLast(Session ses, boolean first) {
+        throw DbException.getUnsupportedException("SCAN");
+    }
+
+    /** {@inheritDoc} */
+    @Override public Cursor findNext(Session ses, SearchRow higherThan, SearchRow last) {
+        throw DbException.throwInternalError();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getColumnIndex(Column col) {
+        return -1;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Column[] getColumns() {
+        return delegate.getColumns();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IndexColumn[] getIndexColumns() {
+        return delegate.getIndexColumns();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IndexType getIndexType() {
+        return TYPE;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Row getRow(Session ses, long key) {
+        return delegate.getRow(ses, key);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCount(Session ses) {
+        return delegate.getRowCount(ses);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCountApproximation() {
+        return delegate.getRowCountApproximation();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Table getTable() {
+        return delegate.getTable();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isRowIdIndex() {
+        return delegate.isRowIdIndex();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean needRebuild() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void remove(Session ses) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void remove(Session ses, Row row) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setSortedInsertMode(boolean sortedInsertMode) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public IndexLookupBatch createLookupBatch(TableFilter filter) {
+        return delegate.createLookupBatch(filter);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void truncate(Session ses) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public Schema getSchema() {
+        return delegate.getSchema();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isHidden() {
+        return delegate.isHidden();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void checkRename() {
+        throw DbException.getUnsupportedException("rename");
+    }
+
+    /** {@inheritDoc} */
+    @Override public ArrayList<DbObject> getChildren() {
+        return delegate.getChildren();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String getComment() {
+        return delegate.getComment();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String getCreateSQL() {
+        return null; // Scan should return null.
+    }
+
+    /** {@inheritDoc} */
+    @Override public String getCreateSQLForCopy(Table tbl, String quotedName) {
+        return delegate.getCreateSQLForCopy(tbl, quotedName);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Database getDatabase() {
+        return delegate.getDatabase();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String getDropSQL() {
+        return delegate.getDropSQL();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getId() {
+        return delegate.getId();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String getSQL() {
+        return delegate.getSQL();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getType() {
+        return delegate.getType();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isTemporary() {
+        return delegate.isTemporary();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeChildrenAndResources(Session ses) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void rename(String newName) {
+        throw DbException.getUnsupportedException("rename");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setComment(String comment) {
+        throw DbException.getUnsupportedException("comment");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setTemporary(boolean temporary) {
+        throw DbException.getUnsupportedException("temporary");
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
index 8d080ae034257..4d5ea4bfec9e2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
@@ -34,22 +34,14 @@
 import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory;
 import org.h2.api.TableEngine;
 import org.h2.command.ddl.CreateTableData;
-import org.h2.engine.Database;
-import org.h2.engine.DbObject;
 import org.h2.engine.Session;
-import org.h2.index.BaseIndex;
-import org.h2.index.Cursor;
 import org.h2.index.Index;
-import org.h2.index.IndexLookupBatch;
 import org.h2.index.IndexType;
 import org.h2.message.DbException;
 import org.h2.result.Row;
 import org.h2.result.SearchRow;
 import org.h2.result.SortOrder;
-import org.h2.schema.Schema;
-import org.h2.table.Column;
 import org.h2.table.IndexColumn;
-import org.h2.table.Table;
 import org.h2.table.TableBase;
 import org.h2.table.TableFilter;
 import org.h2.value.Value;
@@ -857,93 +849,15 @@ public static interface IndexesFactory {
      * Wrapper type for primary key.
      */
     @SuppressWarnings("PackageVisibleInnerClass")
-    static class ScanIndex extends BaseIndex {
+    static class ScanIndex extends GridH2ScanIndex<GridH2IndexBase> {
         /** */
         static final String SCAN_INDEX_NAME_SUFFIX = "__SCAN_";
 
-        /** */
-        private static final IndexType TYPE = IndexType.createScan(false);
-
-        /** */
-        private final GridH2IndexBase delegate;
-
         /**
-         * Constructor.
-         *
-         * @param delegate Index delegate to.
+         * @param delegate Delegate.
          */
-        private ScanIndex(GridH2IndexBase delegate) {
-            this.delegate = delegate;
-        }
-
-        /** {@inheritDoc} */
-        @Override public long getDiskSpaceUsed() {
-            return 0;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void add(Session ses, Row row) {
-            delegate.add(ses, row);
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean canFindNext() {
-            return false;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean canGetFirstOrLast() {
-            return false;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean canScan() {
-            return delegate.canScan();
-        }
-
-        /** {@inheritDoc} */
-        @Override public final void close(Session ses) {
-            // No-op.
-        }
-
-        /** {@inheritDoc} */
-        @Override public void commit(int operation, Row row) {
-            // No-op.
-        }
-
-        /** {@inheritDoc} */
-        @Override public int compareRows(SearchRow rowData, SearchRow compare) {
-            return delegate.compareRows(rowData, compare);
-        }
-
-        /** {@inheritDoc} */
-        @Override public Cursor find(TableFilter filter, SearchRow first, SearchRow last) {
-            return find(filter.getSession(), first, last);
-        }
-
-        /** {@inheritDoc} */
-        @Override public Cursor find(Session ses, SearchRow first, SearchRow last) {
-            return delegate.find(ses, null, null);
-        }
-
-        /** {@inheritDoc} */
-        @Override public Cursor findFirstOrLast(Session ses, boolean first) {
-            throw DbException.getUnsupportedException("SCAN");
-        }
-
-        /** {@inheritDoc} */
-        @Override public Cursor findNext(Session ses, SearchRow higherThan, SearchRow last) {
-            throw DbException.throwInternalError();
-        }
-
-        /** {@inheritDoc} */
-        @Override public int getColumnIndex(Column col) {
-            return -1;
-        }
-
-        /** {@inheritDoc} */
-        @Override public Column[] getColumns() {
-            return delegate.getColumns();
+        public ScanIndex(GridH2IndexBase delegate) {
+            super(delegate);
         }
 
         /** {@inheritDoc} */
@@ -956,164 +870,14 @@ private ScanIndex(GridH2IndexBase delegate) {
             return  mul * baseCost;
         }
 
-        /** {@inheritDoc} */
-        @Override public IndexColumn[] getIndexColumns() {
-            return delegate.getIndexColumns();
-        }
-
-        /** {@inheritDoc} */
-        @Override public IndexType getIndexType() {
-            return TYPE;
-        }
-
         /** {@inheritDoc} */
         @Override public String getPlanSQL() {
             return delegate.getTable().getSQL() + "." + SCAN_INDEX_NAME_SUFFIX;
         }
 
-        /** {@inheritDoc} */
-        @Override public Row getRow(Session ses, long key) {
-            return delegate.getRow(ses, key);
-        }
-
-        /** {@inheritDoc} */
-        @Override public long getRowCount(Session ses) {
-            return delegate.getRowCount(ses);
-        }
-
-        /** {@inheritDoc} */
-        @Override public long getRowCountApproximation() {
-            return delegate.getRowCountApproximation();
-        }
-
-        /** {@inheritDoc} */
-        @Override public Table getTable() {
-            return delegate.getTable();
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean isRowIdIndex() {
-            return delegate.isRowIdIndex();
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean needRebuild() {
-            return false;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void remove(Session ses) {
-            // No-op.
-        }
-
-        /** {@inheritDoc} */
-        @Override public void remove(Session ses, Row row) {
-            // No-op.
-        }
-
-        /** {@inheritDoc} */
-        @Override public void setSortedInsertMode(boolean sortedInsertMode) {
-            // No-op.
-        }
-
-        /** {@inheritDoc} */
-        @Override public IndexLookupBatch createLookupBatch(TableFilter filter) {
-            return delegate.createLookupBatch(filter);
-        }
-
-        /** {@inheritDoc} */
-        @Override public void truncate(Session ses) {
-            // No-op.
-        }
-
-        /** {@inheritDoc} */
-        @Override public Schema getSchema() {
-            return delegate.getSchema();
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean isHidden() {
-            return delegate.isHidden();
-        }
-
-        /** {@inheritDoc} */
-        @Override public void checkRename() {
-            throw DbException.getUnsupportedException("rename");
-        }
-
-        /** {@inheritDoc} */
-        @Override public ArrayList<DbObject> getChildren() {
-            return delegate.getChildren();
-        }
-
-        /** {@inheritDoc} */
-        @Override public String getComment() {
-            return delegate.getComment();
-        }
-
-        /** {@inheritDoc} */
-        @Override public String getCreateSQL() {
-            return null; // Scan should return null.
-        }
-
-        /** {@inheritDoc} */
-        @Override public String getCreateSQLForCopy(Table tbl, String quotedName) {
-            return delegate.getCreateSQLForCopy(tbl, quotedName);
-        }
-
-        /** {@inheritDoc} */
-        @Override public Database getDatabase() {
-            return delegate.getDatabase();
-        }
-
-        /** {@inheritDoc} */
-        @Override public String getDropSQL() {
-            return delegate.getDropSQL();
-        }
-
-        /** {@inheritDoc} */
-        @Override public int getId() {
-            return delegate.getId();
-        }
-
         /** {@inheritDoc} */
         @Override public String getName() {
             return delegate.getName() + SCAN_INDEX_NAME_SUFFIX;
         }
-
-        /** {@inheritDoc} */
-        @Override public String getSQL() {
-            return delegate.getSQL();
-        }
-
-        /** {@inheritDoc} */
-        @Override public int getType() {
-            return delegate.getType();
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean isTemporary() {
-            return delegate.isTemporary();
-        }
-
-        /** {@inheritDoc} */
-        @Override public void removeChildrenAndResources(Session ses) {
-            // No-op.
-        }
-
-        /** {@inheritDoc} */
-        @Override public void rename(String newName) {
-            throw DbException.getUnsupportedException("rename");
-        }
-
-        /** {@inheritDoc} */
-        @Override public void setComment(String comment) {
-            throw DbException.getUnsupportedException("comment");
-        }
-
-        /** {@inheritDoc} */
-        @Override public void setTemporary(boolean temporary) {
-            throw DbException.getUnsupportedException("temporary");
-        }
     }
 }
\ No newline at end of file
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
index 7d4b7f07aa603..951186654d5c8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
@@ -38,9 +38,6 @@ public abstract class GridSqlQuery extends GridSqlStatement implements GridSqlAs
     /** */
     private GridSqlAst offset;
 
-    /** */
-    private boolean distinct;
-
     /**
      * @return Offset.
      */
@@ -55,20 +52,6 @@ public void offset(GridSqlAst offset) {
         this.offset = offset;
     }
 
-    /**
-     * @return Distinct.
-     */
-    public boolean distinct() {
-        return distinct;
-    }
-
-    /**
-     * @param distinct New distinct.
-     */
-    public void distinct(boolean distinct) {
-        this.distinct = distinct;
-    }
-
     /**
      * @return Sort.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index 024529ca83f11..16d7105f2fbcf 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -415,7 +415,7 @@ private GridSqlSelect parseSelect(Select select) {
         res.distinct(select.isDistinct());
 
         Expression where = CONDITION.get(select);
-        res.where(parseExpression(where, false));
+        res.where(parseExpression(where, true));
 
         ArrayList<TableFilter> tableFilters = new ArrayList<>();
 
@@ -447,7 +447,7 @@ private GridSqlSelect parseSelect(Select select) {
             GridSqlElement gridFilter = parseTableFilter(f);
 
             from = from == null ? gridFilter : new GridSqlJoin(from, gridFilter, f.isJoinOuter(),
-                parseExpression(f.getJoinCondition(), false));
+                parseExpression(f.getJoinCondition(), true));
         }
 
         res.from(from);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 277cabc7d21b5..aec0b3664f47b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -207,7 +207,6 @@ public static GridCacheTwoStepQuery split(
         // If we have distributed joins, then we have to optimize all MAP side queries
         // to have a correct join order with respect to batched joins and check if we need
         // distributed joins at all.
-        // TODO Also we need to have a list of table aliases to filter by primary or explicit partitions.
         if (distributedJoins) {
             boolean allCollocated = true;
 
@@ -220,7 +219,7 @@ public static GridCacheTwoStepQuery split(
                 mapSqlQry.query(parse(prepared, true).getSQL());
             }
 
-            // We do not need distributed joins if all MAP queries are colocated.
+            // We do not need distributed joins if all MAP queries are collocated.
             if (allCollocated)
                 distributedJoins = false;
         }
@@ -861,6 +860,7 @@ private void pushDownColumn(
         if (!tblAliases.contains(tblAlias))
             return;
 
+        GridSqlType resType = col.resultType();
         String uniqueColAlias = uniqueColumnAlias(col);
         GridSqlAlias colAlias = cols.get(uniqueColAlias);
 
@@ -874,6 +874,7 @@ private void pushDownColumn(
         col = column(uniqueColAlias);
         // col.tableAlias(wrapAlias.alias());
         col.expressionInFrom(wrapAlias);
+        col.resultType(resType);
 
         prnt.child(childIdx, col);
     }
@@ -1066,7 +1067,7 @@ private void analyzeQueryModel(QueryModel qrym) {
         else if (qrym.type == Type.UNION) {
             // If it is not a UNION ALL, then we have to split because otherwise we can produce duplicates or
             // wrong results for UNION DISTINCT, EXCEPT, INTERSECT queries.
-            if (!qrym.needSplitChild && !qrym.unionAll)
+            if (!qrym.needSplitChild && (!qrym.unionAll || hasOffsetLimit(qrym.<GridSqlUnion>ast())))
                 qrym.needSplitChild = true;
 
             // If we have to split some child SELECT in this UNION, then we have to enforce split
@@ -1150,6 +1151,14 @@ else if (child instanceof GridSqlFunction)
         }
     }
 
+    /**
+     * @param qry Query.
+     * @return {@code true} If we have OFFSET LIMIT.
+     */
+    private static boolean hasOffsetLimit(GridSqlQuery qry) {
+        return qry.limit() != null || qry.offset() != null;
+    }
+
     /**
      * @param select Select to check.
      * @return {@code true} If we need to split this select.
@@ -1158,6 +1167,9 @@ private boolean needSplitSelect(GridSqlSelect select) {
         if (select.distinct())
             return true;
 
+        if (hasOffsetLimit(select))
+            return true;
+
         if (collocatedGrpBy)
             return false;
 
@@ -1304,10 +1316,28 @@ private void splitSelect(
 
         setupParameters(map, mapQry, params);
         map.columns(collectColumns(mapExps));
+        map.sortColumns(mapQry.sort());
+        map.partitioned(hasPartitionedTables(mapQry));
 
         mapSqlQrys.add(map);
     }
 
+    /**
+     * @param ast Map query AST.
+     * @return {@code true} If the given AST has partitioned tables.
+     */
+    private static boolean hasPartitionedTables(GridSqlAst ast) {
+        if (ast instanceof GridSqlTable)
+            return ((GridSqlTable)ast).dataTable().isPartitioned();
+
+        for (int i = 0; i < ast.size(); i++) {
+            if (hasPartitionedTables(ast.child(i)))
+                return true;
+        }
+
+        return false;
+    }
+
     /**
      * @param sqlQry Query.
      * @param qryAst Select AST.
@@ -1333,7 +1363,7 @@ private LinkedHashMap<String,?> collectColumns(List<GridSqlAst> cols) {
             GridSqlType t = col.resultType();
 
             if (t == null)
-                throw new NullPointerException("Column type.");
+                throw new NullPointerException("Column type: " + col);
 
             if (t == GridSqlType.UNKNOWN)
                 throw new IllegalStateException("Unknown type: " + col);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSortColumn.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSortColumn.java
index 8e8947fc38b48..d870ac564f211 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSortColumn.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSortColumn.java
@@ -17,6 +17,13 @@
 
 package org.apache.ignite.internal.processors.query.h2.sql;
 
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.F;
+import org.h2.result.SortOrder;
+import org.h2.table.Column;
+import org.h2.table.IndexColumn;
+import org.h2.table.Table;
+
 /**
  * Sort order for ORDER BY clause.
  */
@@ -46,6 +53,40 @@ public GridSqlSortColumn(int col, boolean asc, boolean nullsFirst, boolean nulls
         this.nullsLast = nullsLast;
     }
 
+    /**
+     * @param tbl Table.
+     * @param sortCols Sort columns.
+     * @return Index columns.
+     */
+    public static IndexColumn[] toIndexColumns(Table tbl, List<GridSqlSortColumn> sortCols) {
+        assert !F.isEmpty(sortCols);
+
+        IndexColumn[] res = new IndexColumn[sortCols.size()];
+
+        for (int i = 0; i < res.length; i++) {
+            GridSqlSortColumn sc = sortCols.get(i);
+
+            Column col = tbl.getColumn(sc.column());
+
+            IndexColumn c = new IndexColumn();
+
+            c.column = col;
+            c.columnName = col.getName();
+
+            c.sortType = sc.asc ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+
+            if (sc.nullsFirst)
+                c.sortType |= SortOrder.NULLS_FIRST;
+
+            if (sc.nullsLast)
+                c.sortType |= SortOrder.NULLS_LAST;
+
+            res[i] = c;
+        }
+
+        return res;
+    }
+
     /**
      * @return Column index.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index f002a5e3b50b4..6416b21d8e18e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -459,8 +459,11 @@ private void onQueryRequest(final ClusterNode node, final GridH2QueryRequest req
             req.isFlagSet(GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS));
 
         final boolean enforceJoinOrder = req.isFlagSet(GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER);
+        final boolean explain = req.isFlagSet(GridH2QueryRequest.FLAG_EXPLAIN);
 
-        for (int i = 1; i < mainCctx.config().getQueryParallelism(); i++) {
+        int segments = explain ? 1 : mainCctx.config().getQueryParallelism();
+
+        for (int i = 1; i < segments; i++) {
             final int segment = i;
 
             ctx.closure().callLocal(
@@ -587,7 +590,6 @@ private void onQueryRequest0(
 
             Connection conn = h2.connectionForSpace(mainCctx.name());
 
-            // Here we enforce join order to have the same behavior on all the nodes.
             setupConnection(conn, distributedJoinMode != OFF, enforceJoinOrder);
 
             GridH2QueryContext.set(qctx);
@@ -610,28 +612,34 @@ private void onQueryRequest0(
                 boolean evt = ctx.event().isRecordable(EVT_CACHE_QUERY_EXECUTED);
 
                 for (GridCacheSqlQuery qry : qrys) {
-                    ResultSet rs = h2.executeSqlQueryWithTimer(mainCctx.name(), conn, qry.query(),
-                        F.asList(qry.parameters()), true,
-                        timeout,
-                        qr.cancels[qryIdx]);
-
-                    if (evt) {
-                        ctx.event().record(new CacheQueryExecutedEvent<>(
-                            node,
-                            "SQL query executed.",
-                            EVT_CACHE_QUERY_EXECUTED,
-                            CacheQueryType.SQL.name(),
-                            mainCctx.namex(),
-                            null,
-                            qry.query(),
-                            null,
-                            null,
-                            qry.parameters(),
-                            node.id(),
-                            null));
-                    }
+                    ResultSet rs = null;
+
+                    // If we are not the target node for this replicated query, just ignore it.
+                    if (qry.node() == null ||
+                        (segmentId == 0 && qry.node().equals(ctx.localNodeId()))) {
+                        rs = h2.executeSqlQueryWithTimer(mainCctx.name(), conn, qry.query(),
+                            F.asList(qry.parameters()), true,
+                            timeout,
+                            qr.cancels[qryIdx]);
+
+                        if (evt) {
+                            ctx.event().record(new CacheQueryExecutedEvent<>(
+                                node,
+                                "SQL query executed.",
+                                EVT_CACHE_QUERY_EXECUTED,
+                                CacheQueryType.SQL.name(),
+                                mainCctx.namex(),
+                                null,
+                                qry.query(),
+                                null,
+                                null,
+                                qry.parameters(),
+                                node.id(),
+                                null));
+                        }
 
-                    assert rs instanceof JdbcResultSet : rs.getClass();
+                        assert rs instanceof JdbcResultSet : rs.getClass();
+                    }
 
                     qr.addResult(qryIdx, qry, node.id(), rs);
 
@@ -751,6 +759,9 @@ private void sendNextPage(NodeResults nodeRess, ClusterNode node, QueryResults q
 
         assert res != null;
 
+        if (res.closed)
+            return;
+
         int page = res.page;
 
         List<Value[]> rows = new ArrayList<>(Math.min(64, pageSize));
@@ -1081,21 +1092,31 @@ private class QueryResult implements AutoCloseable {
          * @param qry Query.
          */
         private QueryResult(ResultSet rs, GridCacheContext<?, ?> cctx, UUID qrySrcNodeId, GridCacheSqlQuery qry) {
-            this.rs = rs;
             this.cctx = cctx;
             this.qry = qry;
             this.qrySrcNodeId = qrySrcNodeId;
             this.cpNeeded = cctx.isLocalNode(qrySrcNodeId);
 
-            try {
-                res = (ResultInterface)RESULT_FIELD.get(rs);
-            }
-            catch (IllegalAccessException e) {
-                throw new IllegalStateException(e); // Must not happen.
+            if (rs != null) {
+                this.rs = rs;
+                try {
+                    res = (ResultInterface)RESULT_FIELD.get(rs);
+                }
+                catch (IllegalAccessException e) {
+                    throw new IllegalStateException(e); // Must not happen.
+                }
+
+                rowCnt = res.getRowCount();
+                cols = res.getVisibleColumnCount();
             }
+            else {
+                this.rs = null;
+                this.res = null;
+                this.cols = -1;
+                this.rowCnt = -1;
 
-            rowCnt = res.getRowCount();
-            cols = res.getVisibleColumnCount();
+                closed = true;
+            }
         }
 
         /**
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
index 6a6e04528f24f..27622bb08db95 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
@@ -22,20 +22,22 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.RandomAccess;
 import java.util.Set;
 import java.util.UUID;
-import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageResponse;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.h2.engine.Session;
 import org.h2.index.BaseIndex;
@@ -44,13 +46,10 @@
 import org.h2.message.DbException;
 import org.h2.result.Row;
 import org.h2.result.SearchRow;
-import org.h2.result.SortOrder;
 import org.h2.table.IndexColumn;
-import org.h2.table.TableFilter;
 import org.h2.value.Value;
 import org.jetbrains.annotations.Nullable;
 
-import static java.util.Collections.emptyIterator;
 import static java.util.Objects.requireNonNull;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_MERGE_TABLE_MAX_SIZE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE;
@@ -66,6 +65,22 @@ public abstract class GridMergeIndex extends BaseIndex {
     /** */
     private static final int PREFETCH_SIZE = getInteger(IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE, 1024);
 
+    /** */
+    private static final AtomicReferenceFieldUpdater<GridMergeIndex, ConcurrentMap> lastPagesUpdater =
+        AtomicReferenceFieldUpdater.newUpdater(GridMergeIndex.class, ConcurrentMap.class, "lastPages");
+
+    static {
+        if (!U.isPow2(PREFETCH_SIZE)) {
+            throw new IllegalArgumentException(IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE + " (" + PREFETCH_SIZE +
+                ") must be positive and a power of 2.");
+        }
+
+        if (PREFETCH_SIZE >= MAX_FETCH_SIZE) {
+            throw new IllegalArgumentException(IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE + " (" + PREFETCH_SIZE +
+                ") must be less than " + IGNITE_SQL_MERGE_TABLE_MAX_SIZE + " (" + MAX_FETCH_SIZE + ").");
+        }
+    }
+
     /** */
     protected final Comparator<SearchRow> firstRowCmp = new Comparator<SearchRow>() {
         @Override public int compare(SearchRow rowInList, SearchRow searchRow) {
@@ -84,14 +99,11 @@ public abstract class GridMergeIndex extends BaseIndex {
         }
     };
 
-    /** All rows number. */
-    private final AtomicInteger expRowsCnt = new AtomicInteger(0);
-
-    /** Remaining rows per source node ID. */
-    private Map<UUID, Counter[]> remainingRows;
+    /** Row source nodes. */
+    private Set<UUID> sources;
 
     /** */
-    private final AtomicBoolean lastSubmitted = new AtomicBoolean();
+    private int pageSize;
 
     /**
      * Will be r/w from query execution thread only, does not need to be threadsafe.
@@ -107,6 +119,9 @@ public abstract class GridMergeIndex extends BaseIndex {
     /** */
     private final GridKernalContext ctx;
 
+    /** */
+    private volatile ConcurrentMap<SourceKey, Integer> lastPages;
+
     /**
      * @param ctx Context.
      * @param tbl Table.
@@ -129,16 +144,6 @@ public GridMergeIndex(GridKernalContext ctx,
      * @param ctx Context.
      */
     protected GridMergeIndex(GridKernalContext ctx) {
-        if (!U.isPow2(PREFETCH_SIZE)) {
-            throw new IllegalArgumentException(IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE + " (" + PREFETCH_SIZE +
-                ") must be positive and a power of 2.");
-        }
-
-        if (PREFETCH_SIZE >= MAX_FETCH_SIZE) {
-            throw new IllegalArgumentException(IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE + " (" + PREFETCH_SIZE +
-                ") must be less than " + IGNITE_SQL_MERGE_TABLE_MAX_SIZE + " (" + MAX_FETCH_SIZE + ").");
-        }
-
         this.ctx = ctx;
 
         fetched = new BlockList<>(PREFETCH_SIZE);
@@ -148,7 +153,7 @@ protected GridMergeIndex(GridKernalContext ctx) {
      * @return Return source nodes for this merge index.
      */
     public Set<UUID> sources() {
-        return remainingRows.keySet();
+        return sources;
     }
 
     /**
@@ -169,17 +174,24 @@ private void checkSourceNodesAlive() {
      * @return {@code true} If this index needs data from the given source node.
      */
     public boolean hasSource(UUID nodeId) {
-        return remainingRows.containsKey(nodeId);
+        return sources.contains(nodeId);
     }
 
     /** {@inheritDoc} */
     @Override public long getRowCount(Session ses) {
-        return expRowsCnt.get();
+        Cursor c = find(ses, null, null);
+
+        long cnt = 0;
+
+        while (c.next())
+            cnt++;
+
+        return cnt;
     }
 
     /** {@inheritDoc} */
     @Override public long getRowCountApproximation() {
-        return getRowCount(null);
+        return 10_000;
     }
 
     /**
@@ -189,27 +201,28 @@ public boolean hasSource(UUID nodeId) {
      * @param segmentsCnt Index segments per table.
      */
     public void setSources(Collection<ClusterNode> nodes, int segmentsCnt) {
-        assert remainingRows == null;
+        assert sources == null;
 
-        remainingRows = U.newHashMap(nodes.size());
+        sources = new HashSet<>();
 
         for (ClusterNode node : nodes) {
-            Counter[] counters = new Counter[segmentsCnt];
-
-            for (int i = 0; i < segmentsCnt; i++)
-                counters[i] = new Counter();
-
-            if (remainingRows.put(node.id(), counters) != null)
-                throw new IllegalStateException("Duplicate node id: " + node.id());
-
+            if (!sources.add(node.id()))
+                throw new IllegalStateException();
         }
     }
 
+    /**
+     * @param pageSize Page size.
+     */
+    public void setPageSize(int pageSize) {
+        this.pageSize = pageSize;
+    }
+
     /**
      * @param queue Queue to poll.
      * @return Next page.
      */
-    private GridResultPage takeNextPage(BlockingQueue<GridResultPage> queue) {
+    private GridResultPage takeNextPage(Pollable<GridResultPage> queue) {
         GridResultPage page;
 
         for (;;) {
@@ -234,16 +247,17 @@ private GridResultPage takeNextPage(BlockingQueue<GridResultPage> queue) {
      * @param iter Current iterator.
      * @return The same or new iterator.
      */
-    protected final Iterator<Value[]> pollNextIterator(BlockingQueue<GridResultPage> queue, Iterator<Value[]> iter) {
-        while (!iter.hasNext()) {
+    protected final Iterator<Value[]> pollNextIterator(Pollable<GridResultPage> queue, Iterator<Value[]> iter) {
+        if (!iter.hasNext()) {
             GridResultPage page = takeNextPage(queue);
 
-            if (page.isLast())
-                return emptyIterator(); // We are done.
-
-            fetchNextPage(page);
+            if (!page.isLast())
+                page.fetchNextPage(); // Failed will throw an exception here.
 
             iter = page.rows();
+
+            // The received iterator must be empty in the dummy last page or on failure.
+            assert iter.hasNext() || page.isDummyLast() || page.isFail();
         }
 
         return iter;
@@ -253,23 +267,18 @@ protected final Iterator<Value[]> pollNextIterator(BlockingQueue<GridResultPage>
      * @param e Error.
      */
     public void fail(final CacheException e) {
-        for (UUID nodeId0 : remainingRows.keySet()) {
-            addPage0(new GridResultPage(null, nodeId0, null) {
-                @Override public boolean isFail() {
-                    return true;
-                }
-
-                @Override public void fetchNextPage() {
-                    throw e;
-                }
-            });
-        }
+        for (UUID nodeId : sources)
+            fail(nodeId, e);
     }
 
     /**
      * @param nodeId Node ID.
+     * @param e Exception.
      */
     public void fail(UUID nodeId, final CacheException e) {
+        if (nodeId == null)
+            nodeId = F.first(sources);
+
         addPage0(new GridResultPage(null, nodeId, null) {
             @Override public boolean isFail() {
                 return true;
@@ -285,91 +294,88 @@ public void fail(UUID nodeId, final CacheException e) {
     }
 
     /**
-     * @param page Page.
+     * @param nodeId Node ID.
+     * @param res Response.
      */
-    public final void addPage(GridResultPage page) {
-        int pageRowsCnt = page.rowsInPage();
+    private void initLastPages(UUID nodeId, GridQueryNextPageResponse res) {
+        int allRows = res.allRows();
 
-        Counter cnt = remainingRows.get(page.source())[page.res.segmentId()];
+        // If the old protocol we send all rows number in the page 0, other pages have -1.
+        // In the new protocol we do not know it and always have -1, except terminating page,
+        // which has -2. Thus we have to init page counters only when we receive positive value
+        // in the first page.
+        if (allRows < 0 || res.page() != 0)
+            return;
 
-        // RemainingRowsCount should be updated before page adding to avoid race
-        // in GridMergeIndexUnsorted cursor iterator
-        int remainingRowsCount;
+        ConcurrentMap<SourceKey,Integer> lp = lastPages;
 
-        int allRows = page.response().allRows();
+        if (lp == null && !lastPagesUpdater.compareAndSet(this, null, lp = new ConcurrentHashMap<>()))
+            lp = lastPages;
 
-        if (allRows != -1) { // Only the first page contains allRows count and is allowed to init counter.
-            assert cnt.state == State.UNINITIALIZED : "Counter is already initialized.";
+        assert pageSize > 0: pageSize;
 
-            remainingRowsCount = cnt.addAndGet(allRows - pageRowsCnt);
+        int lastPage = allRows == 0 ? 0 : (allRows - 1) / pageSize;
 
-            expRowsCnt.addAndGet(allRows);
+        assert lastPage >= 0: lastPage;
 
-            // Add page before setting initialized flag to avoid race condition with adding last page
-            if (pageRowsCnt > 0)
-                addPage0(page);
+        if (lp.put(new SourceKey(nodeId, res.segmentId()), lastPage) != null)
+            throw new IllegalStateException();
+    }
 
-            // We need this separate flag to handle case when the first source contains only one page
-            // and it will signal that all remaining counters are zero and fetch is finished.
-            cnt.state = State.INITIALIZED;
-        }
-        else {
-            remainingRowsCount = cnt.addAndGet(-pageRowsCnt);
+    /**
+     * @param page Page.
+     */
+    private void markLastPage(GridResultPage page) {
+        GridQueryNextPageResponse res = page.response();
 
-            if (pageRowsCnt > 0)
-                addPage0(page);
-        }
+        if (res.allRows() != -2) { // -2 means the last page.
+            UUID nodeId = page.source();
 
-        if (remainingRowsCount == 0) { // Result can be negative in case of race between messages, it is ok.
-            if (cnt.state == State.UNINITIALIZED)
-                return;
+            initLastPages(nodeId, res);
 
-            // Guarantee that finished state possible only if counter is zero and all pages was added
-            cnt.state = State.FINISHED;
+            ConcurrentMap<SourceKey,Integer> lp = lastPages;
 
-            for (Counter[] cntrs : remainingRows.values()) { // Check all the sources.
-                for(int i = 0; i < cntrs.length; i++) {
-                    if (cntrs[i].state != State.FINISHED)
-                        return;
-                }
-            }
+            if (lp == null)
+                return; // It was not initialized --> wait for -2.
 
-            if (lastSubmitted.compareAndSet(false, true)) {
-                addPage0(new GridResultPage(null, page.source(), null) {
-                    @Override public boolean isLast() {
-                        return true;
-                    }
-                });
+            Integer lastPage = lp.get(new SourceKey(nodeId, res.segmentId()));
+
+            if (lastPage == null)
+                return; // This node may use the new protocol --> wait for -2.
+
+            if (lastPage != res.page()) {
+                assert lastPage > res.page();
+
+                return; // This is not the last page.
             }
         }
+
+        page.setLast(true);
     }
 
     /**
      * @param page Page.
      */
-    protected abstract void addPage0(GridResultPage page);
+    public final void addPage(GridResultPage page) {
+        markLastPage(page);
+        addPage0(page);
+    }
 
     /**
-     * @param page Page.
+     * @param lastPage Real last page.
+     * @return Created dummy page.
      */
-    protected void fetchNextPage(GridResultPage page) {
-        assert !page.isLast();
-
-        if(page.isFail())
-            page.fetchNextPage(); //rethrow exceptions
-
-        assert page.res != null;
-
-        Counter[] counters = remainingRows.get(page.source());
+    protected final GridResultPage createDummyLastPage(GridResultPage lastPage) {
+        assert !lastPage.isDummyLast(); // It must be a real last page.
 
-        int segId = page.res.segmentId();
-
-        Counter counter = counters[segId];
-
-        if (counter.get() != 0)
-            page.fetchNextPage();
+        return new GridResultPage(ctx, lastPage.source(), null).setLast(true);
     }
 
+    /**
+     * @param page Page.
+     */
+    protected abstract void addPage0(GridResultPage page);
+
     /** {@inheritDoc} */
     @Override public final Cursor find(Session ses, SearchRow first, SearchRow last) {
         checkBounds(lastEvictedRow, first, last);
@@ -381,11 +387,9 @@ protected void fetchNextPage(GridResultPage page) {
     }
 
     /**
-     * @return {@code true} If we have fetched all the remote rows.
+     * @return {@code true} If we have fetched all the remote rows into a fetched list.
      */
-    public boolean fetchedAll() {
-        return fetchedCnt == expRowsCnt.get();
-    }
+    public abstract boolean fetchedAll();
 
     /**
      * @param lastEvictedRow Last evicted fetched row.
@@ -432,11 +436,6 @@ protected void checkBounds(Row lastEvictedRow, SearchRow first, SearchRow last)
         throw DbException.getUnsupportedException("remove row");
     }
 
-    /** {@inheritDoc} */
-    @Override public double getCost(Session ses, int[] masks, TableFilter[] filters, int filter, SortOrder sortOrder) {
-        return getCostRangeIndex(masks, getRowCountApproximation(), filters, filter, sortOrder, true);
-    }
-
     /** {@inheritDoc} */
     @Override public void remove(Session ses) {
         throw DbException.getUnsupportedException("remove index");
@@ -682,14 +681,6 @@ enum State {
         UNINITIALIZED, INITIALIZED, FINISHED
     }
 
-    /**
-     * Counter with initialization flag.
-     */
-    private static class Counter extends AtomicInteger {
-        /** */
-        volatile State state = State.UNINITIALIZED;
-    }
-
     /**
      */
     private static final class BlockList<Z> extends AbstractList<Z> implements RandomAccess {
@@ -766,4 +757,53 @@ private List<Z> evictFirstBlock() {
             return res;
         }
     }
+
+    /**
+     * Pollable.
+     */
+    protected static interface Pollable<E> {
+        /**
+         * @param timeout Timeout.
+         * @param unit Time unit.
+         * @return Polled value or {@code null} if none.
+         * @throws InterruptedException If interrupted.
+         */
+        E poll(long timeout, TimeUnit unit) throws InterruptedException;
+    }
+
+    /**
+     */
+    private static class SourceKey {
+        final UUID nodeId;
+
+        /** */
+        final int segment;
+
+        /**
+         * @param nodeId Node ID.
+         * @param segment Segment.
+         */
+        SourceKey(UUID nodeId, int segment) {
+            this.nodeId = nodeId;
+            this.segment = segment;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            SourceKey sourceKey = (SourceKey)o;
+
+            if (segment != sourceKey.segment) return false;
+            return nodeId.equals(sourceKey.nodeId);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            int result = nodeId.hashCode();
+            result = 31 * result + segment;
+            return result;
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
index 32c676d37b035..361bb2d5878c0 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
@@ -25,18 +25,24 @@
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.UUID;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2Cursor;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowFactory;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.h2.engine.Session;
 import org.h2.index.Cursor;
 import org.h2.index.IndexType;
 import org.h2.result.Row;
 import org.h2.result.SearchRow;
+import org.h2.result.SortOrder;
 import org.h2.table.IndexColumn;
+import org.h2.table.TableFilter;
 import org.h2.value.Value;
 import org.jetbrains.annotations.Nullable;
 
@@ -47,6 +53,9 @@
  * Sorted index.
  */
 public final class GridMergeIndexSorted extends GridMergeIndex {
+    /** */
+    private static final IndexType TYPE = IndexType.createNonUnique(false);
+
     /** */
     private final Comparator<RowStream> streamCmp = new Comparator<RowStream>() {
         @Override public int compare(RowStream o1, RowStream o2) {
@@ -62,26 +71,33 @@ public final class GridMergeIndexSorted extends GridMergeIndex {
     };
 
     /** */
-    private Map<UUID,RowStream> streamsMap;
+    private Map<UUID,RowStream[]> streamsMap;
 
     /** */
-    private RowStream[] streams;
+    private final Lock lock = new ReentrantLock();
+
+    /** */
+    private final Condition notEmpty = lock.newCondition();
+
+    /** */
+    private GridResultPage failPage;
+
+    /** */
+    private MergeStreamIterator it;
 
     /**
      * @param ctx Kernal context.
      * @param tbl Table.
      * @param name Index name,
-     * @param type Index type.
      * @param cols Columns.
      */
     public GridMergeIndexSorted(
         GridKernalContext ctx,
         GridMergeTable tbl,
         String name,
-        IndexType type,
         IndexColumn[] cols
     ) {
-        super(ctx, tbl, name, type, cols);
+        super(ctx, tbl, name, TYPE, cols);
     }
 
     /** {@inheritDoc} */
@@ -89,33 +105,48 @@ public GridMergeIndexSorted(
         super.setSources(nodes, segmentsCnt);
 
         streamsMap = U.newHashMap(nodes.size());
-        streams = new RowStream[nodes.size()];
+        RowStream[] streams = new RowStream[nodes.size() * segmentsCnt];
 
         int i = 0;
 
         for (ClusterNode node : nodes) {
-            RowStream stream = new RowStream(node.id());
+            RowStream[] segments = new RowStream[segmentsCnt];
 
-            streams[i] = stream;
+            for (int s = 0; s < segmentsCnt; s++)
+                streams[i++] = segments[s] = new RowStream();
 
-            if (streamsMap.put(stream.src, stream) != null)
+            if (streamsMap.put(node.id(), segments) != null)
                 throw new IllegalStateException();
         }
+
+        it = new MergeStreamIterator(streams);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean fetchedAll() {
+        return it.fetchedAll();
     }
 
     /** {@inheritDoc} */
     @Override protected void addPage0(GridResultPage page) {
-        if (page.isLast() || page.isFail()) {
-            // Finish all the streams.
-            for (RowStream stream : streams)
-                stream.addPage(page);
+        if (page.isFail()) {
+            lock.lock();
+
+            try {
+                if (failPage == null) {
+                    failPage = page;
+
+                    notEmpty.signalAll();
+                }
+            }
+            finally {
+                lock.unlock();
+            }
         }
         else {
-            assert page.rowsInPage() > 0;
-
             UUID src = page.source();
 
-            streamsMap.get(src).addPage(page);
+            streamsMap.get(src)[page.segmentId()].addPage(page);
         }
     }
 
@@ -152,9 +183,14 @@ else if (first == null && last == null)
         return new GridH2Cursor(iter);
     }
 
+    /** {@inheritDoc} */
+    @Override public double getCost(Session ses, int[] masks, TableFilter[] filters, int filter, SortOrder sortOrder) {
+        return getCostRangeIndex(masks, getRowCountApproximation(), filters, filter, sortOrder, false);
+    }
+
     /** {@inheritDoc} */
     @Override protected Cursor findInStream(@Nullable SearchRow first, @Nullable SearchRow last) {
-        return new FetchingCursor(first, last, new MergeStreamIterator());
+        return new FetchingCursor(first, last, it);
     }
 
     /**
@@ -165,17 +201,42 @@ private final class MergeStreamIterator implements Iterator<Row> {
         private boolean first = true;
 
         /** */
-        private int off;
+        private volatile int off;
 
         /** */
         private boolean hasNext;
 
+        /** */
+        private final RowStream[] streams;
+
+        /**
+         * @param streams Streams.
+         */
+        MergeStreamIterator(RowStream[] streams) {
+            assert !F.isEmpty(streams);
+
+            this.streams = streams;
+        }
+
+        /**
+         * @return {@code true} If fetched all.
+         */
+        private boolean fetchedAll() {
+            return off == streams.length;
+        }
+
         /**
          *
          */
         private void goFirst() {
+            assert first;
+
+            first = false;
+
             for (int i = 0; i < streams.length; i++) {
-                if (!streams[i].next()) {
+                RowStream s = streams[i];
+
+                if (!s.next()) {
                     streams[i] = null;
                     off++; // Move left bound.
                 }
@@ -183,8 +244,6 @@ private void goFirst() {
 
             if (off < streams.length)
                 Arrays.sort(streams, streamCmp);
-
-            first = false;
         }
 
         /**
@@ -229,31 +288,68 @@ private void goNext() {
     /**
      * Row stream.
      */
-    private final class RowStream {
-        /** */
-        final UUID src;
-
-        /** */
-        final BlockingQueue<GridResultPage> queue = new ArrayBlockingQueue<>(8);
-
+    private final class RowStream implements Pollable<GridResultPage> {
         /** */
         Iterator<Value[]> iter = emptyIterator();
 
         /** */
         Row cur;
 
-        /**
-         * @param src Source.
-         */
-        private RowStream(UUID src) {
-            this.src = src;
-        }
+        /** */
+        GridResultPage nextPage;
 
         /**
          * @param page Page.
          */
         private void addPage(GridResultPage page) {
-            queue.offer(page);
+            assert !page.isFail();
+
+            if (page.isLast() && page.rowsInPage() == 0)
+                page = createDummyLastPage(page); // Terminate.
+
+            lock.lock();
+
+            try {
+                // We can fetch the next page only when we have polled the previous one.
+                assert nextPage == null;
+
+                nextPage = page;
+
+                notEmpty.signalAll();
+            }
+            finally {
+                lock.unlock();
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public GridResultPage poll(long timeout, TimeUnit unit) throws InterruptedException {
+            long nanos = unit.toNanos(timeout);
+
+            lock.lock();
+
+            try {
+                for (;;) {
+                    if (failPage != null)
+                        return failPage;
+
+                    GridResultPage page = nextPage;
+
+                    if (page != null) {
+                        // isLast && !isDummyLast
+                        nextPage = page.isLast() && page.response() != null
+                            ? createDummyLastPage(page) : null; // Terminate with empty iterator.
+
+                        return page;
+                    }
+
+                    if ((nanos = notEmpty.awaitNanos(nanos)) <= 0)
+                        return null;
+                }
+            }
+            finally {
+                lock.unlock();
+            }
         }
 
         /**
@@ -262,7 +358,7 @@ private void addPage(GridResultPage page) {
         private boolean next() {
             cur = null;
 
-            iter = pollNextIterator(queue, iter);
+            iter = pollNextIterator(this, iter);
 
             if (!iter.hasNext())
                 return false;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexUnsorted.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexUnsorted.java
index b69c8986659be..430a687d19ea7 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexUnsorted.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexUnsorted.java
@@ -17,19 +17,24 @@
 
 package org.apache.ignite.internal.processors.query.h2.twostep;
 
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2Cursor;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowFactory;
+import org.h2.engine.Session;
 import org.h2.index.Cursor;
 import org.h2.index.IndexType;
 import org.h2.result.Row;
 import org.h2.result.SearchRow;
+import org.h2.result.SortOrder;
 import org.h2.table.IndexColumn;
+import org.h2.table.TableFilter;
 import org.h2.value.Value;
 
 /**
@@ -37,7 +42,16 @@
  */
 public final class GridMergeIndexUnsorted extends GridMergeIndex {
     /** */
-    private final BlockingQueue<GridResultPage> queue = new LinkedBlockingQueue<>();
+    private static final IndexType TYPE = IndexType.createScan(false);
+
+    /** */
+    private final PollableQueue<GridResultPage> queue = new PollableQueue<>();
+
+    /** */
+    private final AtomicInteger activeSources = new AtomicInteger(-1);
+
+    /** */
+    private Iterator<Value[]> iter = Collections.emptyIterator();
 
     /**
      * @param ctx Context.
@@ -45,7 +59,7 @@ public final class GridMergeIndexUnsorted extends GridMergeIndex {
      * @param name Index name.
      */
     public GridMergeIndexUnsorted(GridKernalContext ctx, GridMergeTable tbl, String name) {
-        super(ctx, tbl, name, IndexType.createScan(false), IndexColumn.wrap(tbl.getColumns()));
+        super(ctx, tbl, name, TYPE, IndexColumn.wrap(tbl.getColumns()));
     }
 
     /**
@@ -63,11 +77,47 @@ private GridMergeIndexUnsorted(GridKernalContext ctx) {
         super(ctx);
     }
 
+    /** {@inheritDoc} */
+    @Override public void setSources(Collection<ClusterNode> nodes, int segmentsCnt) {
+        super.setSources(nodes, segmentsCnt);
+
+        int x = nodes.size() * segmentsCnt;
+
+        assert x > 0: x;
+
+        activeSources.set(x);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean fetchedAll() {
+        int x = activeSources.get();
+
+        assert x >= 0: x; // This method must not be called if the sources were not set.
+
+        return x == 0 && queue.isEmpty();
+    }
+
     /** {@inheritDoc} */
     @Override protected void addPage0(GridResultPage page) {
         assert page.rowsInPage() > 0 || page.isLast() || page.isFail();
 
-        queue.add(page);
+        // Do not add empty page to avoid premature stream termination.
+        if (page.rowsInPage() != 0 || page.isFail())
+            queue.add(page);
+
+        if (page.isLast()) {
+            int x = activeSources.decrementAndGet();
+
+            assert x >= 0: x;
+
+            if (x == 0) // Always terminate with empty iterator.
+                queue.add(createDummyLastPage(page));
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public double getCost(Session ses, int[] masks, TableFilter[] filters, int filter, SortOrder sortOrder) {
+        return getCostRangeIndex(masks, getRowCountApproximation(), filters, filter, sortOrder, true);
     }
 
     /** {@inheritDoc} */
@@ -80,9 +130,6 @@ private GridMergeIndexUnsorted(GridKernalContext ctx) {
     @Override protected Cursor findInStream(SearchRow first, SearchRow last) {
         // This index is unsorted: have to ignore bounds.
         return new FetchingCursor(null, null, new Iterator<Row>() {
-            /** */
-            Iterator<Value[]> iter = Collections.emptyIterator();
-
             @Override public boolean hasNext() {
                 iter = pollNextIterator(queue, iter);
 
@@ -98,4 +145,10 @@ private GridMergeIndexUnsorted(GridKernalContext ctx) {
             }
         });
     }
+
+    /**
+     */
+    private static class PollableQueue<X> extends LinkedBlockingQueue<X> implements Pollable<X> {
+        // No-op.
+    }
 }
\ No newline at end of file
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeTable.java
index 1489021007195..f7495c03b76dd 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeTable.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeTable.java
@@ -18,35 +18,55 @@
 package org.apache.ignite.internal.processors.query.h2.twostep;
 
 import java.util.ArrayList;
-import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2ScanIndex;
+import org.apache.ignite.internal.util.typedef.F;
 import org.h2.command.ddl.CreateTableData;
 import org.h2.engine.Session;
 import org.h2.index.Index;
 import org.h2.index.IndexType;
 import org.h2.message.DbException;
 import org.h2.result.Row;
+import org.h2.result.SortOrder;
 import org.h2.table.IndexColumn;
 import org.h2.table.TableBase;
+import org.h2.table.TableFilter;
 
 /**
  * Merge table for distributed queries.
  */
 public class GridMergeTable extends TableBase {
     /** */
-    private final GridKernalContext ctx;
-
-    /** */
-    private final GridMergeIndex idx;
+    private ArrayList<Index> idxs;
 
     /**
      * @param data Data.
-     * @param ctx Kernal context.
      */
-    public GridMergeTable(CreateTableData data, GridKernalContext ctx) {
+    public GridMergeTable(CreateTableData data) {
         super(data);
+    }
+
+    /**
+     * @param idxs Indexes.
+     */
+    public void indexes(ArrayList<Index> idxs) {
+        assert !F.isEmpty(idxs);
+
+        this.idxs = idxs;
+    }
 
-        this.ctx = ctx;
-        idx = new GridMergeIndexUnsorted(ctx, this, "merge_scan");
+    /**
+     * @return Merge index.
+     */
+    public GridMergeIndex getMergeIndex() {
+        return (GridMergeIndex)idxs.get(idxs.size() - 1); // Sorted index must be the last.
+    }
+
+    /**
+     * @param idx Index.
+     * @return Scan index.
+     */
+    public static GridH2ScanIndex<GridMergeIndex> createScanIndex(GridMergeIndex idx) {
+        return new ScanIndex(idx);
     }
 
     /** {@inheritDoc} */
@@ -56,7 +76,7 @@ public GridMergeTable(CreateTableData data, GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public void close(Session ses) {
-        idx.close(ses);
+        // No-op.
     }
 
     /** {@inheritDoc} */
@@ -96,8 +116,8 @@ public GridMergeTable(CreateTableData data, GridKernalContext ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public GridMergeIndex getScanIndex(Session session) {
-        return idx;
+    @Override public Index getScanIndex(Session session) {
+        return idxs.get(0); // Must be always at 0.
     }
 
     /** {@inheritDoc} */
@@ -107,7 +127,7 @@ public GridMergeTable(CreateTableData data, GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public ArrayList<Index> getIndexes() {
-        return null;
+        return idxs;
     }
 
     /** {@inheritDoc} */
@@ -137,12 +157,12 @@ public GridMergeTable(CreateTableData data, GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public long getRowCount(Session ses) {
-        return idx.getRowCount(ses);
+        return getScanIndex(ses).getRowCount(ses);
     }
 
     /** {@inheritDoc} */
     @Override public long getRowCountApproximation() {
-        return idx.getRowCountApproximation();
+        return getScanIndex(null).getRowCountApproximation();
     }
 
     /** {@inheritDoc} */
@@ -154,4 +174,24 @@ public GridMergeTable(CreateTableData data, GridKernalContext ctx) {
     @Override public void checkRename() {
         throw DbException.getUnsupportedException("rename");
     }
+
+    /**
+     * Scan index wrapper.
+     */
+    private static class ScanIndex extends GridH2ScanIndex<GridMergeIndex> {
+        /**
+         * @param delegate Delegate.
+         */
+        public ScanIndex(GridMergeIndex delegate) {
+            super(delegate);
+        }
+
+        /** {@inheritDoc} */
+        @Override public double getCost(Session session, int[] masks, TableFilter[] filters, int filter,
+            SortOrder sortOrder) {
+            long rows = getRowCountApproximation();
+
+            return getCostRangeIndex(masks, rows, filters, filter, sortOrder, true);
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 4cae6acd9c510..8d3ece9e19fe7 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -67,6 +67,7 @@
 import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlSortColumn;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlType;
 import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest;
 import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse;
@@ -86,6 +87,7 @@
 import org.h2.command.ddl.CreateTableData;
 import org.h2.engine.Session;
 import org.h2.index.Cursor;
+import org.h2.index.Index;
 import org.h2.jdbc.JdbcConnection;
 import org.h2.jdbc.JdbcResultSet;
 import org.h2.jdbc.JdbcStatement;
@@ -97,6 +99,7 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
+import static java.util.Collections.singletonList;
 import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
 import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
 import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.setupConnection;
@@ -114,6 +117,12 @@ public class GridReduceQueryExecutor {
     /** */
     private static final IgniteProductVersion DISTRIBUTED_JOIN_SINCE = IgniteProductVersion.fromString("1.7.0");
 
+    /** */
+    private static final String MERGE_INDEX_UNSORTED = "merge_scan";
+
+    /** */
+    private static final String MERGE_INDEX_SORTED = "merge_sorted";
+
     /** */
     private GridKernalContext ctx;
 
@@ -354,7 +363,7 @@ private void onNextPage(final ClusterNode node, GridQueryNextPageResponse msg) {
 
         if (msg.retry() != null)
             retry(r, msg.retry(), node.id());
-        else if (msg.allRows() != -1) // Only the first page contains row count.
+        else if (msg.page() == 0) // Do count down on each first page received.
             r.latch.countDown();
     }
 
@@ -518,7 +527,7 @@ public Iterator<List<?>> query(
             Map<ClusterNode, IntArray> partsMap = null;
 
             if (qry.isLocal())
-                nodes = Collections.singleton(ctx.discovery().localNode());
+                nodes = singletonList(ctx.discovery().localNode());
             else {
                 if (isPreloadingActive(cctx, extraSpaces)) {
                     if (cctx.isReplicated())
@@ -542,17 +551,17 @@ public Iterator<List<?>> query(
                         "We must be on a client node.";
 
                     // Select random data node to run query on a replicated data or get EXPLAIN PLAN from a single node.
-                    nodes = Collections.singleton(F.rand(nodes));
+                    nodes = singletonList(F.rand(nodes));
                 }
             }
 
-            final Collection<ClusterNode> finalNodes = nodes;
-
             int tblIdx = 0;
 
             final boolean skipMergeTbl = !qry.explain() && qry.skipMergeTable();
 
-            final int segmentsPerIndex = cctx.config().getQueryParallelism();
+            final int segmentsPerIndex = qry.explain() ? 1 : cctx.config().getQueryParallelism();
+
+            int replicatedQrysCnt = 0;
 
             for (GridCacheSqlQuery mapQry : qry.mapQueries()) {
                 GridMergeIndex idx;
@@ -567,19 +576,33 @@ public Iterator<List<?>> query(
                         throw new IgniteException(e);
                     }
 
-                    idx = tbl.getScanIndex(null);
+                    idx = tbl.getMergeIndex();
 
                     fakeTable(r.conn, tblIdx++).innerTable(tbl);
                 }
                 else
                     idx = GridMergeIndexUnsorted.createDummy(ctx);
 
-                idx.setSources(nodes, segmentsPerIndex);
+                // If the query has only replicated tables, we have to run it on a single node only.
+                if (!mapQry.isPartitioned()) {
+                    ClusterNode node = F.rand(nodes);
+
+                    mapQry.node(node.id());
+
+                    replicatedQrysCnt++;
+
+                    idx.setSources(singletonList(node), 1); // Replicated tables can have only 1 segment.
+                }
+                else
+                    idx.setSources(nodes, segmentsPerIndex);
+
+                idx.setPageSize(r.pageSize);
 
                 r.idxs.add(idx);
             }
 
-            r.latch = new CountDownLatch(r.idxs.size() * nodes.size() * segmentsPerIndex);
+            r.latch = new CountDownLatch(
+                (r.idxs.size() - replicatedQrysCnt) * nodes.size() * segmentsPerIndex + replicatedQrysCnt);
 
             runs.put(qryReqId, r);
 
@@ -607,6 +630,8 @@ public Iterator<List<?>> query(
                 final boolean oldStyle = minNodeVer.compareToIgnoreTimestamp(DISTRIBUTED_JOIN_SINCE) < 0;
                 final boolean distributedJoins = qry.distributedJoins();
 
+                final Collection<ClusterNode> finalNodes = nodes;
+
                 cancel.set(new Runnable() {
                     @Override public void run() {
                         send(finalNodes, new GridQueryCancelRequest(qryReqId), null, false);
@@ -627,6 +652,9 @@ public Iterator<List<?>> query(
                 if (qry.isLocal())
                     flags |= GridH2QueryRequest.FLAG_IS_LOCAL;
 
+                if (qry.explain())
+                    flags |= GridH2QueryRequest.FLAG_EXPLAIN;
+
                 if (send(nodes,
                     oldStyle ?
                         new GridQueryRequest(qryReqId,
@@ -724,7 +752,7 @@ public Iterator<List<?>> query(
                                 r.conn,
                                 rdc.query(),
                                 F.asList(rdc.parameters()),
-                                false,
+                                false, // The statement will cache some extra thread local objects.
                                 timeoutMillis,
                                 cancel);
 
@@ -782,19 +810,6 @@ public Iterator<List<?>> query(
         }
     }
 
-    /**
-     * @param idxs Merge indexes.
-     * @return {@code true} If all remote data was fetched.
-     */
-    private static boolean allIndexesFetched(List<GridMergeIndex> idxs) {
-        for (int i = 0; i <  idxs.size(); i++) {
-            if (!idxs.get(i).fetchedAll())
-                return false;
-        }
-
-        return true;
-    }
-
     /**
      * Returns true if the exception is triggered by query cancel.
      *
@@ -1256,6 +1271,7 @@ private static Map<UUID, int[]> convert(Map<ClusterNode, IntArray> m) {
      * @return Table.
      * @throws IgniteCheckedException If failed.
      */
+    @SuppressWarnings("unchecked")
     private GridMergeTable createMergeTable(JdbcConnection conn, GridCacheSqlQuery qry, boolean explain)
         throws IgniteCheckedException {
         try {
@@ -1290,7 +1306,31 @@ private GridMergeTable createMergeTable(JdbcConnection conn, GridCacheSqlQuery q
             else
                 data.columns = planColumns();
 
-            return new GridMergeTable(data, ctx);
+            boolean sortedIndex = !F.isEmpty(qry.sortColumns());
+
+            GridMergeTable tbl = new GridMergeTable(data);
+
+            ArrayList<Index> idxs = new ArrayList<>(2);
+
+            if (explain) {
+                idxs.add(new GridMergeIndexUnsorted(ctx, tbl,
+                    sortedIndex ? MERGE_INDEX_SORTED : MERGE_INDEX_UNSORTED));
+            }
+            else if (sortedIndex) {
+                List<GridSqlSortColumn> sortCols = (List<GridSqlSortColumn>)qry.sortColumns();
+
+                GridMergeIndexSorted sortedMergeIdx = new GridMergeIndexSorted(ctx, tbl, MERGE_INDEX_SORTED,
+                    GridSqlSortColumn.toIndexColumns(tbl, sortCols));
+
+                idxs.add(GridMergeTable.createScanIndex(sortedMergeIdx));
+                idxs.add(sortedMergeIdx);
+            }
+            else
+                idxs.add(new GridMergeIndexUnsorted(ctx, tbl, MERGE_INDEX_UNSORTED));
+
+            tbl.indexes(idxs);
+
+            return tbl;
         }
         catch (Exception e) {
             U.closeQuiet(conn);
@@ -1407,22 +1447,17 @@ void state(Object o, @Nullable UUID nodeId) {
             while (latch.getCount() != 0) // We don't need to wait for all nodes to reply.
                 latch.countDown();
 
+            CacheException e = o instanceof CacheException ? (CacheException) o : null;
+
             for (GridMergeIndex idx : idxs) // Fail all merge indexes.
-                idx.fail(nodeId, o instanceof CacheException ? (CacheException) o : null);
+                idx.fail(nodeId, e);
         }
 
         /**
          * @param e Error.
          */
         void disconnected(CacheException e) {
-            if (!state.compareAndSet(null, e))
-                return;
-
-            while (latch.getCount() != 0) // We don't need to wait for all nodes to reply.
-                latch.countDown();
-
-            for (GridMergeIndex idx : idxs) // Fail all merge indexes.
-                idx.fail(e);
+            state(e, null);
         }
     }
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridResultPage.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridResultPage.java
index 205733093586c..103084e662052 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridResultPage.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridResultPage.java
@@ -40,7 +40,7 @@ public class GridResultPage {
     private final UUID src;
 
     /** */
-    protected final GridQueryNextPageResponse res;
+    private final GridQueryNextPageResponse res;
 
     /** */
     private final int rowsInPage;
@@ -48,6 +48,9 @@ public class GridResultPage {
     /** */
     private Iterator<Value[]> rows;
 
+    /** */
+    private boolean last;
+
     /**
      * @param ctx Kernal context.
      * @param src Source.
@@ -119,10 +122,28 @@ public boolean isFail() {
     }
 
     /**
-     * @return {@code true} If this is a dummy last page for all the sources.
+     * @return {@code true} If this is either a real last page for a source or
+     *      a dummy terminating page with no rows.
      */
     public boolean isLast() {
-        return false;
+        return last;
+    }
+
+    /**
+     * @return {@code true} If it is a dummy last page.
+     */
+    public boolean isDummyLast() {
+        return last && res == null;
+    }
+
+    /**
+     * @param last Last page for a source.
+     * @return {@code this}.
+     */
+    public GridResultPage setLast(boolean last) {
+        this.last = last;
+
+        return this;
     }
 
     /**
@@ -152,6 +173,13 @@ public UUID source() {
         return src;
     }
 
+    /**
+     * @return Segment ID.
+     */
+    public int segmentId() {
+        return res.segmentId();
+    }
+
     /**
      * @return Response.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
index 0ad534c5b6d79..0aa788bc82819 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
@@ -48,17 +48,22 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
      * Map query will not destroy context until explicit query cancel request will be received because distributed join
      * requests can be received.
      */
-    public static int FLAG_DISTRIBUTED_JOINS = 1;
+    public static final int FLAG_DISTRIBUTED_JOINS = 1;
 
     /**
      * Remote map query executor will enforce join order for the received map queries.
      */
-    public static int FLAG_ENFORCE_JOIN_ORDER = 1 << 1;
+    public static final int FLAG_ENFORCE_JOIN_ORDER = 1 << 1;
 
     /**
      * Restrict distributed joins range-requests to local index segments. Range requests to other nodes will not be sent.
      */
-    public static int FLAG_IS_LOCAL = 1 << 2;
+    public static final int FLAG_IS_LOCAL = 1 << 2;
+
+    /**
+     * If it is an EXPLAIN command.
+     */
+    public static final int FLAG_EXPLAIN = 1 << 3;
 
     /** */
     private long reqId;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
index 81c28a323fca8..842b9471f0255 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
@@ -395,10 +395,7 @@ public void testUserDefinedFunction() throws IgniteCheckedException {
 
         Collection<List<?>> res = qry.getAll();
 
-        if (cacheMode() == REPLICATED)
-            assertEquals(1, res.size());
-        else
-            assertEquals(gridCount(), res.size());
+        assertEquals(1, res.size());
 
         List<?> row = res.iterator().next();
 
@@ -410,10 +407,7 @@ public void testUserDefinedFunction() throws IgniteCheckedException {
 
         res = qry.getAll();
 
-        if (cacheMode() == REPLICATED)
-            assertEquals(1, res.size());
-        else
-            assertEquals(gridCount(), res.size());
+        assertEquals(1, res.size());
 
         row = res.iterator().next();
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 69f66a59cae61..8eae54950214b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -36,13 +36,12 @@
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.affinity.AffinityKeyMapped;
 import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.query.h2.twostep.GridMergeIndex;
 import org.apache.ignite.internal.util.GridRandom;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
@@ -153,6 +152,70 @@ public void testOffsetLimit() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSortedMergeIndex() throws Exception {
+        IgniteCache<Integer,Value> c = ignite(0).getOrCreateCache(cacheConfig("v", true,
+            Integer.class, Value.class));
+
+        try {
+            GridTestUtils.setFieldValue(null, GridMergeIndex.class, "PREFETCH_SIZE", 8);
+
+            Random rnd = new GridRandom();
+
+            int cnt = 1000;
+
+            for (int i = 0; i < cnt; i++) {
+                c.put(i, new Value(
+                    rnd.nextInt(5) == 0 ? null: rnd.nextInt(100),
+                    rnd.nextInt(8) == 0 ? null: rnd.nextInt(2000)));
+            }
+
+            List<List<?>> plan = c.query(new SqlFieldsQuery(
+                "explain select snd from Value order by fst desc")).getAll();
+            String rdcPlan = (String)plan.get(1).get(0);
+
+            assertTrue(rdcPlan.contains("merge_sorted"));
+            assertTrue(rdcPlan.contains("/* index sorted */"));
+
+            plan = c.query(new SqlFieldsQuery(
+                "explain select snd from Value")).getAll();
+            rdcPlan = (String)plan.get(1).get(0);
+
+            assertTrue(rdcPlan.contains("merge_scan"));
+            assertFalse(rdcPlan.contains("/* index sorted */"));
+
+            for (int i = 0; i < 10; i++) {
+                X.println(" --> " + i);
+
+                List<List<?>> res = c.query(new SqlFieldsQuery(
+                    "select fst from Value order by fst").setPageSize(5)
+                ).getAll();
+
+                assertEquals(cnt, res.size());
+
+                Integer p = null;
+
+                for (List<?> row : res) {
+                    Integer x = (Integer)row.get(0);
+
+                    if (x != null) {
+                        if (p != null)
+                            assertTrue(x + " >= " + p,  x >= p);
+
+                        p = x;
+                    }
+                }
+            }
+        }
+        finally {
+            GridTestUtils.setFieldValue(null, GridMergeIndex.class, "PREFETCH_SIZE", 1024);
+
+            c.destroy();
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -848,13 +911,44 @@ public void testIndexSegmentationPartitionedReplicated() throws Exception {
 
             String select0 = "select o.name n1, p.name n2 from \"pers\".Person2 p, \"org\".Organization o where p.orgId = o._key";
 
-            final SqlFieldsQuery qry = new SqlFieldsQuery(select0);
+            SqlFieldsQuery qry = new SqlFieldsQuery(select0);
 
             qry.setDistributedJoins(true);
 
             List<List<?>> results = c1.query(qry).getAll();
 
             assertEquals(2, results.size());
+
+            select0 += " order by n2 desc";
+
+            qry = new SqlFieldsQuery(select0);
+
+            qry.setDistributedJoins(true);
+
+            results = c1.query(qry).getAll();
+
+            assertEquals(2, results.size());
+
+            assertEquals("p2", results.get(0).get(1));
+            assertEquals("p1", results.get(1).get(1));
+
+            // Test for replicated subquery with aggregate.
+            select0 = "select p.name " +
+                "from \"pers\".Person2 p, " +
+                "(select max(_key) orgId from \"org\".Organization) o " +
+                "where p.orgId = o.orgId";
+
+            X.println("Plan: \n" +
+                c1.query(new SqlFieldsQuery("explain " + select0).setDistributedJoins(true)).getAll());
+
+            qry = new SqlFieldsQuery(select0);
+
+            qry.setDistributedJoins(true);
+
+            results = c1.query(qry).getAll();
+
+            assertEquals(1, results.size());
+            assertEquals("p2", results.get(0).get(0));
         }
         finally {
             c1.destroy();
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
index f8526a841e9cf..30dd80d1b882c 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
@@ -249,11 +249,13 @@ public void testBigQuery() throws Exception {
         X.println(bigQry);
         X.println();
 
-        X.println("Plan: " + pCache.query(new SqlFieldsQuery("EXPLAIN " + bigQry)
+        X.println("   Plan: \n" + pCache.query(new SqlFieldsQuery("EXPLAIN " + bigQry)
             .setDistributedJoins(distributedJoins())).getAll());
 
         List<List<?>> res = compareQueryRes0(pCache, bigQry, distributedJoins(), new Object[0], Ordering.RANDOM);
 
+        X.println("   Result size: " + res.size());
+
         assertTrue(!res.isEmpty()); // Ensure we set good testing data at database.
     }
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index b417b0ad1ed7c..0c74f127fbff0 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -106,6 +106,7 @@
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2TableSelfTest;
 import org.apache.ignite.internal.processors.query.h2.sql.BaseH2CompareQueryTest;
 import org.apache.ignite.internal.processors.query.h2.sql.GridQueryParsingTest;
+import org.apache.ignite.internal.processors.query.h2.sql.H2CompareBigQueryDistributedJoinsTest;
 import org.apache.ignite.internal.processors.query.h2.sql.H2CompareBigQueryTest;
 import org.apache.ignite.spi.communication.tcp.GridOrderedMessageCancelSelfTest;
 import org.apache.ignite.testframework.IgniteTestSuite;
@@ -214,6 +215,7 @@ public static TestSuite suite() throws Exception {
         // Ignite cache and H2 comparison.
         suite.addTestSuite(BaseH2CompareQueryTest.class);
         suite.addTestSuite(H2CompareBigQueryTest.class);
+        suite.addTestSuite(H2CompareBigQueryDistributedJoinsTest.class);
 
         // Cache query metrics.
         suite.addTestSuite(CacheLocalQueryMetricsSelfTest.class);
diff --git a/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql b/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
index 8d42d44bc1f49..64b7e2ac02ca1 100644
--- a/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
+++ b/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
@@ -26,21 +26,25 @@ from (
 	
 	select  date, orderId, rootOrderId, refOrderId as origOrderId, archSeq, alias
 	from "part".ReplaceOrder where alias='CUSTOM'
-  ) co, "part".OrderParams op
-  where co.date = op.date and co.orderId = op.orderId and co.archSeq = -- TODO: replace with 'dateToLong(co.date)+archSeq'.
-	(
-	  select max(archSeq) -- TODO: replace with 'dateToLong(co.date)+archSeq'.
-	  from (
-		  select  date, orderId, rootOrderId, origOrderId, archSeq, alias 
-		  from "part".CustOrder where alias='CUSTOM'
-		  
-		  union all
-		  
-		  select  date, orderId, rootOrderId, refOrderId as origOrderId, archSeq, alias
-		  from "part".ReplaceOrder where alias='CUSTOM'
-	  ) 
-	  where origOrderId = co.origOrderId and date = co.date
-	) and co.alias='CUSTOM'
+  ) co,
+  "part".OrderParams op,
+    (
+      select origOrderId, date, max(archSeq) maxArchSeq
+      from (
+          select  date, orderId, rootOrderId, origOrderId, archSeq, alias
+          from "part".CustOrder where alias='CUSTOM'
+
+          union all
+
+          select  date, orderId, rootOrderId, refOrderId as origOrderId, archSeq, alias
+          from "part".ReplaceOrder where alias='CUSTOM'
+      )
+      group by origOrderId, date
+   ) h
+  where co.date = op.date and co.orderId = op.orderId
+    and h.origOrderId = co.origOrderId and h.date = co.date
+    and co.archSeq = h.maxArchSeq
+	and co.alias='CUSTOM'
 ) cop 
 inner join (
   select e.date, e.rootOrderId as eRootOrderId, e.rootOrderId, sum(e.execShares) as execShares, 

From bcb139822afa148a7ea3fbb3eecc274f308070f6 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 10 Mar 2017 15:51:38 +0700
Subject: [PATCH 043/516] IGNITE-4717 VisorClearTask minor fix. (cherry picked
 from commit d4b87f4)

---
 .../visor/cache/VisorCacheClearTask.java      | 57 ++++++++++++++++++-
 1 file changed, 56 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
index 0c8476fa91b10..ce74f1760fa27 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.resources.JobContextResource;
@@ -145,4 +146,58 @@ private boolean callAsync(IgniteFuture<Integer> fut, int idx) {
             return S.toString(VisorCacheClearJob.class, this);
         }
     }
-}
\ No newline at end of file
+
+    /**
+     * Callable to get cache size.
+     *
+     * @deprecated This class needed only for compatibility.
+     */
+    @GridInternal @Deprecated
+    private static class VisorCacheSizeCallable implements IgniteCallable<Integer> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteCache cache;
+
+        /**
+         * @param cache Cache to take size from.
+         */
+        private VisorCacheSizeCallable(IgniteCache cache) {
+            this.cache = cache;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer call() throws Exception {
+            return cache.size(CachePeekMode.PRIMARY);
+        }
+    }
+
+    /**
+     * Callable to clear cache.
+     *
+     * @deprecated This class needed only for compatibility.
+     */
+    @GridInternal @Deprecated
+    private static class VisorCacheClearCallable implements IgniteCallable<Integer> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteCache cache;
+
+        /**
+         * @param cache Cache to clear.
+         */
+        private VisorCacheClearCallable(IgniteCache cache) {
+            this.cache = cache;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer call() throws Exception {
+            cache.clear();
+
+            return 0;
+        }
+    }
+}

From 590b82d817595b82a0706e332c545e4746fafbc2 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 10 Mar 2017 15:51:38 +0700
Subject: [PATCH 044/516] IGNITE-4717 VisorClearTask minor fix. (cherry picked
 from commit d4b87f4)

---
 .../visor/cache/VisorCacheClearTask.java      | 57 ++++++++++++++++++-
 1 file changed, 56 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
index 0c8476fa91b10..ce74f1760fa27 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.resources.JobContextResource;
@@ -145,4 +146,58 @@ private boolean callAsync(IgniteFuture<Integer> fut, int idx) {
             return S.toString(VisorCacheClearJob.class, this);
         }
     }
-}
\ No newline at end of file
+
+    /**
+     * Callable to get cache size.
+     *
+     * @deprecated This class needed only for compatibility.
+     */
+    @GridInternal @Deprecated
+    private static class VisorCacheSizeCallable implements IgniteCallable<Integer> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteCache cache;
+
+        /**
+         * @param cache Cache to take size from.
+         */
+        private VisorCacheSizeCallable(IgniteCache cache) {
+            this.cache = cache;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer call() throws Exception {
+            return cache.size(CachePeekMode.PRIMARY);
+        }
+    }
+
+    /**
+     * Callable to clear cache.
+     *
+     * @deprecated This class needed only for compatibility.
+     */
+    @GridInternal @Deprecated
+    private static class VisorCacheClearCallable implements IgniteCallable<Integer> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteCache cache;
+
+        /**
+         * @param cache Cache to clear.
+         */
+        private VisorCacheClearCallable(IgniteCache cache) {
+            this.cache = cache;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer call() throws Exception {
+            cache.clear();
+
+            return 0;
+        }
+    }
+}

From 0ed4fdacc7cc8cf41b7726fc4a42db1a43241285 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 14 Mar 2017 15:50:03 +0300
Subject: [PATCH 045/516] IGNITE-4761: Fix ServiceProcessor hanging on node
 stop. This closes #1602.

---
 .../service/GridServiceProcessor.java         | 15 ++--
 .../GridServiceProcessorStopSelfTest.java     | 75 +++++++++++++++++++
 2 files changed, 83 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 4eeafed421214..6bcfd65cb59e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -315,6 +315,8 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
 
         busyLock.block();
 
+        U.shutdownNow(GridServiceProcessor.class, depExe, log);
+
         if (!ctx.clientNode())
             ctx.event().removeLocalEventListener(topLsnr);
 
@@ -352,8 +354,6 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
             }
         }
 
-        U.shutdownNow(GridServiceProcessor.class, depExe, log);
-
         Exception err = new IgniteCheckedException("Operation has been cancelled (node is stopping).");
 
         cancelFutures(depFuts, err);
@@ -1399,7 +1399,7 @@ private class ServiceEntriesListener implements CacheEntryUpdatedListener<Object
                 return;
 
             try {
-                depExe.submit(new BusyRunnable() {
+                depExe.submit(new DepRunnable() {
                     @Override public void run0() {
                         onSystemCacheUpdated(deps);
                     }
@@ -1586,7 +1586,7 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                 else
                     topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
 
-                depExe.submit(new BusyRunnable() {
+                depExe.submit(new DepRunnable() {
                     @Override public void run0() {
                         ClusterNode oldest = ctx.discovery().oldestAliveCacheServerNode(topVer);
 
@@ -1794,12 +1794,15 @@ private void undeploy(String name) {
     /**
      *
      */
-    private abstract class BusyRunnable implements Runnable {
+    private abstract class DepRunnable implements Runnable {
         /** {@inheritDoc} */
         @Override public void run() {
             if (!busyLock.enterBusy())
                 return;
 
+            // Won't block ServiceProcessor stopping process.
+            busyLock.leaveBusy();
+
             svcName.set(null);
 
             try {
@@ -1812,8 +1815,6 @@ private abstract class BusyRunnable implements Runnable {
                     throw t;
             }
             finally {
-                busyLock.leaveBusy();
-
                 svcName.set(null);
             }
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
index 92b18abed143c..ea0ba51b60954 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
@@ -20,10 +20,16 @@
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteServices;
 import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.services.Service;
@@ -90,6 +96,75 @@ public void testStopDuringDeployment() throws Exception {
         fut.get();
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStopDuringHangedDeployment() throws Exception {
+        final CountDownLatch depLatch = new CountDownLatch(1);
+
+        final CountDownLatch finishLatch = new CountDownLatch(1);
+
+        final IgniteEx node0 = startGrid(0);
+        final IgniteEx node1 = startGrid(1);
+        final IgniteEx node2 = startGrid(2);
+
+        final IgniteCache<Object, Object> cache = node2.getOrCreateCache(new CacheConfiguration<Object, Object>("def")
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL));
+
+        node0.services().deployNodeSingleton("myService", new TestServiceImpl());
+
+        // Guarantee lock owner will never left topology unexpectedly.
+        final Integer lockKey = keyForNode(node2.affinity("def"), new AtomicInteger(1),
+            node2.cluster().localNode());
+
+        // Lock to hold topology version undone.
+        final Lock lock = cache.lock(lockKey);
+
+        // Try to change topology once service has deployed.
+        IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                depLatch.await();
+
+                node1.close();
+
+                return null;
+            }
+        }, "top-change-thread");
+
+        // Stop node on unstable topology.
+        GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                depLatch.await();
+
+                Thread.sleep(1000);
+
+                node0.close();
+
+                finishLatch.countDown();
+
+                return null;
+            }
+        }, "stopping-node-thread");
+
+        assertNotNull(node0.services().service("myService"));
+
+        // Freeze topology changing
+        lock.lock();
+
+        depLatch.countDown();
+
+        boolean wait = finishLatch.await(15, TimeUnit.SECONDS);
+
+        if (!wait)
+            U.dumpThreads(log);
+
+        assertTrue("Deploy future isn't completed", wait);
+
+        fut.get();
+
+        Ignition.stopAll(true);
+    }
+
     /**
      * Simple map service.
      */

From 6633da64d57eadacf30bf437a9c3a3d205903dcd Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Tue, 14 Mar 2017 16:40:58 +0300
Subject: [PATCH 046/516] IGNITE-3386 Reverted: "Reentrant lock is lost when
 lock owner leaves topology"

---
 .../main/java/org/apache/ignite/Ignite.java   |  1 -
 .../DataStructuresProcessor.java              |  3 +-
 .../datastructures/GridCacheLockImpl.java     | 17 +-----
 .../internal/GridCacheRecreateLockTest.java   | 58 -------------------
 .../IgniteComputeGridTestSuite.java           |  2 -
 5 files changed, 3 insertions(+), 78 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java
index 7f8974c20fd9e..0de08d56584a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignite.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java
@@ -527,7 +527,6 @@ public IgniteSemaphore semaphore(String name, int cnt, boolean failoverSafe, boo
      *      all threads on other nodes waiting to acquire lock are interrupted.
      * @param fair If {@code True}, fair lock will be created.
      * @param create Boolean flag indicating whether data structure should be created if does not exist.
-     *      Will re-create lock if the node that stored the lock left topology and there are no backups left.
      * @return ReentrantLock for the given name.
      * @throws IgniteException If reentrant lock could not be fetched or created.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
index 698efd9d7665d..1cad22f33e5ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
@@ -1396,8 +1396,7 @@ public IgniteLock reentrantLock(final String name, final boolean failoverSafe, f
                         name,
                         key,
                         reentrantLockView,
-                        dsCacheCtx,
-                        create);
+                        dsCacheCtx);
 
                     dsMap.put(key, reentrantLock0);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
index 1cf78faa5bd4e..3ab7289237a63 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java
@@ -105,9 +105,6 @@ public final class GridCacheLockImpl implements GridCacheLockEx, Externalizable
     /** Flag indicating that every operation on this lock should be interrupted. */
     private volatile boolean interruptAll;
 
-    /** Re-create flag. */
-    private volatile boolean reCreate;
-
     /**
      * Empty constructor required by {@link Externalizable}.
      */
@@ -525,14 +522,7 @@ protected boolean compareAndSetGlobalState(final int expVal, final int newVal,
                                 GridCacheLockState val = lockView.get(key);
 
                                 if (val == null)
-                                    if (reCreate) {
-                                        val = new GridCacheLockState(0, ctx.nodeId(), 0, failoverSafe, fair);
-
-                                        lockView.put(key, val);
-                                    }
-                                    else
-                                        throw new IgniteCheckedException("Failed to find reentrant lock with " +
-                                            "the given name: " + name);
+                                    throw new IgniteCheckedException("Failed to find reentrant lock with given name: " + name);
 
                                 final long newThreadID = newThread.getId();
 
@@ -1058,14 +1048,12 @@ protected IgniteConditionObject(String name, ConditionObject object) {
      * @param key Reentrant lock key.
      * @param lockView Reentrant lock projection.
      * @param ctx Cache context.
-     * @param reCreate If {@code true} reentrant lock will be re-created in case it is not in cache.
      */
     @SuppressWarnings("unchecked")
     public GridCacheLockImpl(String name,
         GridCacheInternalKey key,
         IgniteInternalCache<GridCacheInternalKey, GridCacheLockState> lockView,
-        GridCacheContext ctx,
-        boolean reCreate) {
+        GridCacheContext ctx) {
         assert name != null;
         assert key != null;
         assert ctx != null;
@@ -1075,7 +1063,6 @@ public GridCacheLockImpl(String name,
         this.key = key;
         this.lockView = lockView;
         this.ctx = ctx;
-        this.reCreate = reCreate;
 
         log = ctx.logger(getClass());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
deleted file mode 100644
index ae850f7789745..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridCacheRecreateLockTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.apache.ignite.internal;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLock;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.testframework.junits.common.GridCommonTest;
-
-/**
- * Create lock after owner node left topology test
- */
-@GridCommonTest(group = "Kernal Self")
-public class GridCacheRecreateLockTest extends GridCommonAbstractTest {
-    /**
-     * @throws IgniteCheckedException If failed.
-     */
-    public void testLockOwnerLeavesGrid() throws Exception {
-        final Ignite ignite = startNodeAndLock("node1");
-
-        new Thread(new Runnable() {
-            @Override public void run() {
-                try {
-                    Thread.sleep(2000);
-                }
-                catch (InterruptedException e) {
-                    e.printStackTrace();
-                }
-
-                ignite.close();
-            }
-        }).start();
-
-        startNodeAndLock("node2");
-    }
-
-    /**
-     * @param name Grid name.
-     * @return Started Ignite instance.
-     */
-    private Ignite startNodeAndLock(String name) throws Exception {
-        IgniteConfiguration cfg = new IgniteConfiguration();
-        cfg.setGridName(name);
-
-        Ignite ignite = Ignition.start(cfg);
-
-        IgniteLock lock = ignite.reentrantLock("lock", true, true, true);
-
-        System.out.println("acquiring lock");
-
-        lock.lock();
-
-        System.out.println("acquired lock");
-
-        return ignite;
-    }
-}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 7c8b6a94c8f0d..8a501fdb801d6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -22,7 +22,6 @@
 import org.apache.ignite.internal.GridAffinityNoCacheSelfTest;
 import org.apache.ignite.internal.GridAffinitySelfTest;
 import org.apache.ignite.internal.GridAlwaysFailoverSpiFailSelfTest;
-import org.apache.ignite.internal.GridCacheRecreateLockTest;
 import org.apache.ignite.internal.GridCancelOnGridStopSelfTest;
 import org.apache.ignite.internal.GridCancelUnusedJobSelfTest;
 import org.apache.ignite.internal.GridCancelledJobsMetricsSelfTest;
@@ -153,7 +152,6 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TaskNodeRestartTest.class);
         suite.addTestSuite(IgniteRoundRobinErrorAfterClientReconnectTest.class);
         suite.addTestSuite(PublicThreadpoolStarvationTest.class);
-        suite.addTestSuite(GridCacheRecreateLockTest.class);
 
         return suite;
     }

From 841887ee3ace1cbe988168e1027ae32500079456 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Thu, 16 Mar 2017 15:42:46 +0300
Subject: [PATCH 047/516] Merge remote-tracking branch ignite-1.7.9 into
 ignite-1.9.2

Conflicts:
	docs/RELEASE_NOTES.txt
	docs/community/RELEASE_NOTES.txt
	modules/compatibility/src/test/java/org/gridgain/grid/compatibility/tests/GridCompatibilityAbstractTest.java
	modules/core/src/main/resources/gridgain.properties
	modules/visor/src/main/scala/org/gridgain/visor/gui/tabs/sql/VisorSqlViewerTab.scala
---
 .../ignite/internal/visor/cache/VisorCacheClearTask.java     | 1 +
 .../internal/processors/query/h2/IgniteH2Indexing.java       | 5 -----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
index b588c18efd4af..62b4762be615d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.resources.JobContextResource;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 740b737032ffb..407819746d4b5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -1286,11 +1286,6 @@ public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable
 
         runs.put(run.id(), run);
 
-        GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL, spaceName,
-            U.currentTimeMillis(), null, true);
-
-        runs.put(run.id(), run);
-
         try {
             ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, cancel);
 

From d124004d8b0396a44c26f4c35c263a15880f508c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 17 Mar 2017 14:57:48 +0300
Subject: [PATCH 048/516] IGNITE-4473 - Client should re-try connection attempt
 in case of concurrent network failure.

---
 .../internal/GridKernalGatewayImpl.java       |   8 +-
 .../apache/ignite/internal/IgniteKernal.java  | 120 +++++-
 .../IgniteNeedReconnectException.java         |  40 ++
 .../discovery/GridDiscoveryManager.java       |  24 ++
 .../GridCachePartitionExchangeManager.java    |  25 +-
 .../dht/GridDhtAssignmentFetchFuture.java     |  14 +-
 .../GridDhtPartitionsExchangeFuture.java      |  48 ++-
 .../service/GridServiceProcessor.java         |  86 ++--
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 201 ++++++++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   5 +
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |   8 +
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   9 +
 .../IgniteClientReconnectCacheTest.java       |   7 +-
 .../internal/IgniteClientRejoinTest.java      | 378 ++++++++++++++++++
 .../tcp/TcpClientDiscoverySpiSelfTest.java    |  48 ++-
 .../IgniteClientReconnectTestSuite.java       |   2 +
 16 files changed, 929 insertions(+), 94 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
index fe8c580ca63ea..036954a3a280d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
@@ -44,7 +44,7 @@ public class GridKernalGatewayImpl implements GridKernalGateway, Serializable {
 
     /** */
     @GridToStringExclude
-    private IgniteFutureImpl<?> reconnectFut;
+    private volatile IgniteFutureImpl<?> reconnectFut;
 
     /** */
     private final AtomicReference<GridKernalState> state = new AtomicReference<>(GridKernalState.STOPPED);
@@ -149,6 +149,12 @@ public GridKernalGatewayImpl(String gridName) {
 
     /** {@inheritDoc} */
     @Override public GridFutureAdapter<?> onDisconnected() {
+        if (state.get() == GridKernalState.DISCONNECTED) {
+            assert reconnectFut != null;
+
+            return (GridFutureAdapter<?>)reconnectFut.internalFuture();
+        }
+
         GridFutureAdapter<?> fut = new GridFutureAdapter<>();
 
         reconnectFut = new IgniteFutureImpl<>(fut);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 8fda72fc18419..25f7884c889b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -250,6 +250,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     /** Periodic starvation check interval. */
     private static final long PERIODIC_STARVATION_CHECK_FREQ = 1000 * 30;
 
+    /** Force complete reconnect future. */
+    private static final Object STOP_RECONNECT = new Object();
+
     /** */
     @GridToStringExclude
     private GridKernalContextImpl ctx;
@@ -327,6 +330,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     @GridToStringExclude
     private final AtomicBoolean stopGuard = new AtomicBoolean();
 
+    /** */
+    private final ReconnectState reconnectState = new ReconnectState();
+
     /**
      * No-arg constructor is required by externalization.
      */
@@ -930,6 +936,8 @@ public void start(final IgniteConfiguration cfg,
             // Notify IO manager the second so further components can send and receive messages.
             ctx.io().onKernalStart();
 
+            boolean recon = false;
+
             // Callbacks.
             for (GridComponent comp : ctx) {
                 // Skip discovery manager.
@@ -940,10 +948,24 @@ public void start(final IgniteConfiguration cfg,
                 if (comp instanceof GridIoManager)
                     continue;
 
-                if (!skipDaemon(comp))
-                    comp.onKernalStart();
+                if (!skipDaemon(comp)) {
+                    try {
+                        comp.onKernalStart();
+                    }
+                    catch (IgniteNeedReconnectException e) {
+                        assert ctx.discovery().reconnectSupported();
+
+                        if (log.isDebugEnabled())
+                            log.debug("Failed to start node components on node start, will wait for reconnect: " + e);
+
+                        recon = true;
+                    }
+                }
             }
 
+            if (recon)
+                reconnectState.waitFirstReconnect();
+
             // Register MBeans.
             registerKernalMBean();
             registerLocalNodeMBean();
@@ -3274,6 +3296,8 @@ private void unguard() {
     public void onDisconnected() {
         Throwable err = null;
 
+        reconnectState.waitPreviousReconnect();
+
         GridFutureAdapter<?> reconnectFut = ctx.gateway().onDisconnected();
 
         if (reconnectFut == null) {
@@ -3282,9 +3306,18 @@ public void onDisconnected() {
             return;
         }
 
-        IgniteFuture<?> userFut = new IgniteFutureImpl<>(reconnectFut);
+        IgniteFutureImpl<?> curFut = (IgniteFutureImpl<?>)ctx.cluster().get().clientReconnectFuture();
+
+        IgniteFuture<?> userFut;
 
-        ctx.cluster().get().clientReconnectFuture(userFut);
+        // In case of previous reconnect did not finish keep reconnect future.
+        if (curFut != null && curFut.internalFuture() == reconnectFut)
+            userFut = curFut;
+        else {
+            userFut = new IgniteFutureImpl<>(reconnectFut);
+
+            ctx.cluster().get().clientReconnectFuture(userFut);
+        }
 
         ctx.disconnected(true);
 
@@ -3337,30 +3370,53 @@ public void onReconnected(final boolean clusterRestarted) {
         try {
             ctx.disconnected(false);
 
-            GridCompoundFuture<?, ?> reconnectFut = new GridCompoundFuture<>();
+            GridCompoundFuture curReconnectFut = reconnectState.curReconnectFut = new GridCompoundFuture<>();
+
+            reconnectState.reconnectDone = new GridFutureAdapter<>();
 
             for (GridComponent comp : ctx.components()) {
                 IgniteInternalFuture<?> fut = comp.onReconnected(clusterRestarted);
 
                 if (fut != null)
-                    reconnectFut.add((IgniteInternalFuture)fut);
+                    curReconnectFut.add(fut);
             }
 
-            reconnectFut.add((IgniteInternalFuture)ctx.cache().context().exchange().reconnectExchangeFuture());
+            curReconnectFut.add(ctx.cache().context().exchange().reconnectExchangeFuture());
+
+            curReconnectFut.markInitialized();
 
-            reconnectFut.markInitialized();
+            final GridFutureAdapter reconnectDone = reconnectState.reconnectDone;
 
-            reconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
+            curReconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
                 @Override public void apply(IgniteInternalFuture<?> fut) {
                     try {
-                        fut.get();
+                        Object res = fut.get();
+
+                        if (res == STOP_RECONNECT)
+                            return;
 
                         ctx.gateway().onReconnected();
+
+                        reconnectState.firstReconnectFut.onDone();
                     }
                     catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to reconnect, will stop node", e);
+                        if (!X.hasCause(e, IgniteNeedReconnectException.class,
+                            IgniteClientDisconnectedCheckedException.class)) {
+                            U.error(log, "Failed to reconnect, will stop node.", e);
+
+                            reconnectState.firstReconnectFut.onDone(e);
 
-                        close();
+                            close();
+                        }
+                        else {
+                            assert ctx.discovery().reconnectSupported();
+
+                            U.error(log, "Failed to finish reconnect, will retry [locNodeId=" + ctx.localNodeId() +
+                                ", err=" + e.getMessage() + ']');
+                        }
+                    }
+                    finally {
+                        reconnectDone.onDone();
                     }
                 }
             });
@@ -3519,6 +3575,46 @@ public void dumpDebugInfo() {
         }
     }
 
+    /**
+     *
+     */
+    private class ReconnectState {
+        /** */
+        private final GridFutureAdapter firstReconnectFut = new GridFutureAdapter();
+
+        /** */
+        private GridCompoundFuture<?, Object> curReconnectFut;
+
+        /** */
+        private GridFutureAdapter<?> reconnectDone;
+
+        /**
+         * @throws IgniteCheckedException If failed.
+         */
+        void waitFirstReconnect() throws IgniteCheckedException {
+            firstReconnectFut.get();
+        }
+
+        /**
+         *
+         */
+        void waitPreviousReconnect() {
+            if (curReconnectFut != null && !curReconnectFut.isDone()) {
+                assert reconnectDone != null;
+
+                curReconnectFut.onDone(STOP_RECONNECT);
+
+                try {
+                    reconnectDone.get();
+                }
+                catch (IgniteCheckedException ignote) {
+                    // No-op.
+                }
+            }
+
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IgniteKernal.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
new file mode 100644
index 0000000000000..61ab5762b95ac
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Indicates that node should try reconnect to cluster.
+ */
+public class IgniteNeedReconnectException extends IgniteCheckedException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * @param locNode Local node.
+     * @param cause Cause.
+     */
+    public IgniteNeedReconnectException(ClusterNode locNode, @Nullable Throwable cause) {
+        super("Local node need try to reconnect [locNodeId=" + locNode.id() + ']', cause);
+
+        assert locNode.isClient();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 9aa4db1e0042f..2ec10705bedbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -112,6 +112,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -1890,6 +1891,29 @@ public void failNode(UUID nodeId, @Nullable String warning) {
         }
     }
 
+    /**
+     * @return {@code True} if local node client and discovery SPI supports reconnect.
+     */
+    public boolean reconnectSupported() {
+        DiscoverySpi spi = getSpi();
+
+        return ctx.clientNode() && (spi instanceof TcpDiscoverySpi) &&
+            !(((TcpDiscoverySpi) spi).isClientReconnectDisabled());
+    }
+
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() {
+        assert reconnectSupported();
+
+        DiscoverySpi discoverySpi = getSpi();
+
+        ((TcpDiscoverySpi)discoverySpi).reconnect();
+    }
+
     /**
      * Updates topology version if current version is smaller than updated.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7f11dc45aacb7..92142c0731240 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -43,6 +43,7 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
@@ -447,6 +448,15 @@ else if (m instanceof GridDhtPartitionDemandMessage)
                     else
                         U.warn(log, "Still waiting for initial partition map exchange [fut=" + fut + ']');
                 }
+                catch (IgniteNeedReconnectException e) {
+                    throw e;
+                }
+                catch (Exception e) {
+                    if (fut.reconnectOnError(e))
+                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                    throw e;
+                }
             }
 
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
@@ -1690,6 +1700,12 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                         dumpedObjects++;
                                     }
                                 }
+                                catch (Exception e) {
+                                    if (exchFut.reconnectOnError(e))
+                                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                                    throw e;
+                                }
                             }
 
 
@@ -1829,7 +1845,14 @@ else if (r != null) {
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
                 }
-                catch (IgniteClientDisconnectedCheckedException e) {
+                catch (IgniteClientDisconnectedCheckedException | IgniteNeedReconnectException e) {
+                    assert cctx.discovery().reconnectSupported();
+
+                    U.warn(log,"Local node failed to complete partition map exchange due to " +
+                        "network issues, will try to reconnect to cluster", e);
+
+                    cctx.discovery().reconnect();
+
                     return;
                 }
                 catch (IgniteCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index ab8e863e90e7d..6425bc141245d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -17,15 +17,16 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.Queue;
 import java.util.UUID;
-import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
@@ -34,6 +35,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -202,8 +204,14 @@ private void requestFromNextNode() {
                         "continue to another node): " + node);
                 }
                 catch (IgniteCheckedException e) {
-                    U.error(log0, "Failed to request affinity assignment from remote node (will " +
-                        "continue to another node): " + node, e);
+                    if (ctx.discovery().reconnectSupported() && X.hasCause(e, IOException.class)) {
+                        onDone(new IgniteNeedReconnectException(ctx.localNode(), e));
+
+                        return;
+                    }
+
+                    U.warn(log0, "Failed to request affinity assignment from remote node (will " +
+                        "continue to another node): " + node);
                 }
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index e945de958a339..d4f95e5b55d05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -32,6 +33,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
@@ -39,6 +41,7 @@
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -54,7 +57,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
@@ -65,7 +67,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -506,10 +508,17 @@ public void init() throws IgniteInterruptedCheckedException {
 
             throw e;
         }
+        catch (IgniteNeedReconnectException e) {
+            onDone(e);
+        }
         catch (Throwable e) {
-            U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else {
+                U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
 
-            onDone(e);
+                onDone(e);
+            }
 
             if (e instanceof Error)
                 throw (Error)e;
@@ -1297,7 +1306,10 @@ private void onAllReceived(boolean discoThread) {
             }
         }
         catch (IgniteCheckedException e) {
-            onDone(e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else
+                onDone(e);
         }
     }
 
@@ -1314,8 +1326,15 @@ private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
         }
         catch (IgniteCheckedException e) {
             if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) {
-                log.debug("Failed to send full partition map to node, node left grid " +
-                    "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+                if (log.isDebugEnabled())
+                    log.debug("Failed to send full partition map to node, node left grid " +
+                        "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+
+                return;
+            }
+
+            if (reconnectOnError(e)) {
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
 
                 return;
             }
@@ -1641,6 +1660,12 @@ public void onNodeLeft(final ClusterNode node) {
                             }
                         }
                     }
+                    catch (Exception e) {
+                        if (reconnectOnError(e))
+                            onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+                        else
+                            throw e;
+                    }
                     finally {
                         leaveBusy();
                     }
@@ -1652,6 +1677,15 @@ public void onNodeLeft(final ClusterNode node) {
         }
     }
 
+    /**
+     * @param e Exception.
+     * @return {@code True} if local node should try reconnect in case of error.
+     */
+    public boolean reconnectOnError(Throwable e) {
+        return X.hasCause(e, IOException.class, IgniteClientDisconnectedCheckedException.class) &&
+            cctx.discovery().reconnectSupported();
+    }
+
     /** {@inheritDoc} */
     @Override public int compareTo(GridDhtPartitionsExchangeFuture fut) {
         return exchId.compareTo(fut.exchId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6bcfd65cb59e0..bd815189fb5ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1498,60 +1498,60 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
         }
     }
 
-        /**
-         * Deployment callback.
-         *
-         * @param dep Service deployment.
-         * @param topVer Topology version.
-         */
-        private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
-            // Retry forever.
-            try {
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+    /**
+     * Deployment callback.
+     *
+     * @param dep Service deployment.
+     * @param topVer Topology version.
+     */
+    private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
+        // Retry forever.
+        try {
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                // If topology version changed, reassignment will happen from topology event.
-                if (newTopVer.equals(topVer))
-                    reassign(dep, topVer);
-            }
-            catch (IgniteCheckedException e) {
-                if (!(e instanceof ClusterTopologyCheckedException))
-                    log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
+            // If topology version changed, reassignment will happen from topology event.
+            if (newTopVer.equals(topVer))
+                reassign(dep, topVer);
+        }
+        catch (IgniteCheckedException e) {
+            if (!(e instanceof ClusterTopologyCheckedException))
+                log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
 
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                if (!newTopVer.equals(topVer)) {
-                    assert newTopVer.compareTo(topVer) > 0;
+            if (!newTopVer.equals(topVer)) {
+                assert newTopVer.compareTo(topVer) > 0;
 
-                    // Reassignment will happen from topology event.
-                    return;
-                }
+                // Reassignment will happen from topology event.
+                return;
+            }
 
-                ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
-                    private IgniteUuid id = IgniteUuid.randomUuid();
+            ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
+                private IgniteUuid id = IgniteUuid.randomUuid();
 
-                    private long start = System.currentTimeMillis();
+                private long start = System.currentTimeMillis();
 
-                    @Override public IgniteUuid timeoutId() {
-                        return id;
-                    }
+                @Override public IgniteUuid timeoutId() {
+                    return id;
+                }
 
-                    @Override public long endTime() {
-                        return start + RETRY_TIMEOUT;
-                    }
+                @Override public long endTime() {
+                    return start + RETRY_TIMEOUT;
+                }
 
-                    @Override public void onTimeout() {
-                        if (!busyLock.enterBusy())
-                            return;
+                @Override public void onTimeout() {
+                    if (!busyLock.enterBusy())
+                        return;
 
-                        try {
-                            // Try again.
-                            onDeployment(dep, topVer);
-                        }
-                        finally {
-                            busyLock.leaveBusy();
-                        }
+                    try {
+                        // Try again.
+                        onDeployment(dep, topVer);
                     }
-                });
+                    finally {
+                        busyLock.leaveBusy();
+                    }
+                }
+            });
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 95e2cda4f6907..02ba56a884b33 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -129,6 +129,9 @@ class ClientImpl extends TcpDiscoveryImpl {
     /** */
     private static final Object SPI_RECONNECT_FAILED = "SPI_RECONNECT_FAILED";
 
+    /** */
+    private static final Object SPI_RECONNECT = "SPI_RECONNECT";
+
     /** Remote nodes. */
     private final ConcurrentMap<UUID, TcpDiscoveryNode> rmtNodes = new ConcurrentHashMap8<>();
 
@@ -808,6 +811,11 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
             log);
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        msgWorker.addMessage(SPI_RECONNECT);
+    }
+
     /** {@inheritDoc} */
     @Override public void brakeConnection() {
         SocketStream sockStream = msgWorker.currSock;
@@ -879,9 +887,12 @@ private class SocketReader extends IgniteSpiThread {
         /** */
         private UUID rmtNodeId;
 
+        /** */
+        private CountDownLatch stopReadLatch;
+
         /**
          */
-        protected SocketReader() {
+        SocketReader() {
             super(spi.ignite().name(), "tcp-client-disco-sock-reader", log);
         }
 
@@ -889,7 +900,7 @@ protected SocketReader() {
          * @param sockStream Socket.
          * @param rmtNodeId Rmt node id.
          */
-        public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
+        void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             synchronized (mux) {
                 this.sockStream = sockStream;
 
@@ -899,6 +910,31 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             }
         }
 
+        /**
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceStopRead() throws InterruptedException {
+            CountDownLatch stopReadLatch;
+
+            synchronized (mux) {
+                SocketStream stream = sockStream;
+
+                if (stream == null)
+                    return;
+
+                this.stopReadLatch = stopReadLatch = new CountDownLatch(1);
+
+                U.closeQuiet(stream.socket());
+
+                this.sockStream = null;
+                this.rmtNodeId = null;
+
+                mux.notifyAll();
+            }
+
+            stopReadLatch.await();
+        }
+
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException {
             while (!isInterrupted()) {
@@ -906,6 +942,12 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
                 UUID rmtNodeId;
 
                 synchronized (mux) {
+                    if (stopReadLatch != null) {
+                        stopReadLatch.countDown();
+
+                        stopReadLatch = null;
+                    }
+
                     if (this.sockStream == null) {
                         mux.wait();
 
@@ -1007,18 +1049,21 @@ private class SocketWriter extends IgniteSpiThread {
         private final Queue<TcpDiscoveryAbstractMessage> queue = new ArrayDeque<>();
 
         /** */
-        private final long socketTimeout;
+        private final long sockTimeout;
 
         /** */
         private TcpDiscoveryAbstractMessage unackedMsg;
 
+        /** */
+        private CountDownLatch forceLeaveLatch;
+
         /**
          *
          */
-        protected SocketWriter() {
+        SocketWriter() {
             super(spi.ignite().name(), "tcp-client-disco-sock-writer", log);
 
-            socketTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
+            sockTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
                 spi.getSocketTimeout();
         }
 
@@ -1033,6 +1078,29 @@ private void sendMessage(TcpDiscoveryAbstractMessage msg) {
             }
         }
 
+        /**
+         * Sends {@link TcpDiscoveryNodeLeftMessage} and closes socket.
+         *
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceLeave() throws InterruptedException {
+            CountDownLatch forceLeaveLatch;
+
+            synchronized (mux) {
+                // If writer was stopped.
+                if (sock == null)
+                    return;
+
+                this.forceLeaveLatch = forceLeaveLatch = new CountDownLatch(1);
+
+                unackedMsg = null;
+
+                mux.notifyAll();
+            }
+
+            forceLeaveLatch.await();
+        }
+
         /**
          * @param sock Socket.
          * @param clientAck {@code True} is server supports client message acknowlede.
@@ -1089,13 +1157,41 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                         continue;
                     }
 
-                    msg = queue.poll();
+                    if (forceLeaveLatch != null) {
+                        msg = new TcpDiscoveryNodeLeftMessage(getLocalNodeId());
 
-                    if (msg == null) {
-                        mux.wait();
+                        msg.client(true);
+
+                        try {
+                            spi.writeToSocket(
+                                sock,
+                                msg,
+                                sockTimeout);
+                        }
+                        catch (IOException | IgniteCheckedException e) {
+                            if (log.isDebugEnabled()) {
+                                log.debug("Failed to send TcpDiscoveryNodeLeftMessage on force leave [msg=" + msg +
+                                    ", err=" + e.getMessage() + ']');
+                            }
+                        }
+
+                        U.closeQuiet(sock);
+
+                        this.sock = null;
+
+                        clear();
 
                         continue;
                     }
+                    else {
+                        msg = queue.poll();
+
+                        if (msg == null) {
+                            mux.wait();
+
+                            continue;
+                        }
+                    }
                 }
 
                 for (IgniteInClosure<TcpDiscoveryAbstractMessage> msgLsnr : spi.sndMsgLsnrs)
@@ -1115,7 +1211,7 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     spi.writeToSocket(
                         sock,
                         msg,
-                        socketTimeout);
+                        sockTimeout);
 
                     msg = null;
 
@@ -1165,10 +1261,30 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     synchronized (mux) {
                         if (sock == this.sock)
                             this.sock = null; // Connection has dead.
+
+                        clear();
                     }
                 }
             }
         }
+
+        /**
+         *
+         */
+        private void clear() {
+            assert Thread.holdsLock(mux);
+
+            queue.clear();
+            unackedMsg = null;
+
+            CountDownLatch forceLeaveLatch = this.forceLeaveLatch;
+
+            if (forceLeaveLatch != null) {
+                this.forceLeaveLatch = null;
+
+                forceLeaveLatch.countDown();
+            }
+        }
     }
 
     /**
@@ -1413,6 +1529,38 @@ else if (msg == SPI_STOP) {
                         else
                             leaveLatch.countDown();
                     }
+                    else if (msg == SPI_RECONNECT) {
+                        if (state == CONNECTED) {
+                            if (reconnector != null) {
+                                reconnector.cancel();
+                                reconnector.join();
+
+                                reconnector = null;
+                            }
+
+                            sockWriter.forceLeave();
+                            sockReader.forceStopRead();
+
+                            currSock = null;
+
+                            queue.clear();
+
+                            onDisconnected();
+
+                            notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
+
+                            UUID newId = UUID.randomUUID();
+
+                            U.quietAndWarn(log, "Local node will try to reconnect to cluster with new id due " +
+                                "to network problems [newId=" + newId +
+                                ", prevId=" + locNode.id() +
+                                ", locNode=" + locNode+ ']');
+
+                            locNode.onClientDisconnected(newId);
+
+                            tryJoin();
+                        }
+                    }
                     else if (msg instanceof TcpDiscoveryNodeFailedMessage &&
                         ((TcpDiscoveryNodeFailedMessage)msg).failedNodeId().equals(locNode.id())) {
                         TcpDiscoveryNodeFailedMessage msg0 = (TcpDiscoveryNodeFailedMessage)msg;
@@ -1495,20 +1643,7 @@ else if (msg == SPI_RECONNECT_FAILED) {
                                         ", failMsg=" + forceFailMsg + ']');
                                 }
 
-                                state = DISCONNECTED;
-
-                                nodeAdded = false;
-
-                                IgniteClientDisconnectedCheckedException err =
-                                    new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
-                                    "client node disconnected.");
-
-                                for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
-                                    GridFutureAdapter<Boolean> fut = e.getValue();
-
-                                    if (pingFuts.remove(e.getKey(), fut))
-                                        fut.onDone(err);
-                                }
+                                onDisconnected();
 
                                 notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
                             }
@@ -1603,6 +1738,26 @@ else if (discoMsg instanceof TcpDiscoveryCheckFailedMessage)
             }
         }
 
+        /**
+         *
+         */
+        private void onDisconnected() {
+            state = DISCONNECTED;
+
+            nodeAdded = false;
+
+            IgniteClientDisconnectedCheckedException err =
+                new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
+                    "client node disconnected.");
+
+            for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
+                GridFutureAdapter<Boolean> fut = e.getValue();
+
+                if (pingFuts.remove(e.getKey(), fut))
+                    fut.onDone(err);
+            }
+        }
+
         /**
          * @throws InterruptedException If interrupted.
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 4600be094c840..afd1c2ba7fc66 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1609,6 +1609,11 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
         throw new UnsupportedOperationException();
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        throw new UnsupportedOperationException("Reconnect is not supported for server.");
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteSpiThread workerThread() {
         return msgWorker;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index f199c20e17edb..84c2ff28afef0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -258,6 +259,13 @@ protected static String threadStatus(Thread t) {
         return t.isAlive() ? "alive" : "dead";
     }
 
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public abstract void reconnect() throws IgniteSpiException;
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      * <p>
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 00ae97d1e302c..a2a47feb31486 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1926,6 +1926,15 @@ boolean isSslEnabled() {
         return ignite().configuration().getSslContextFactory() != null;
     }
 
+    /**
+     * Force reconnect to cluster.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() throws IgniteSpiException {
+        impl.reconnect();
+    }
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 0f0165b0f04f9..6cdf465978907 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -700,9 +700,12 @@ public void testReconnectInitialExchangeInProgress() throws Exception {
                 try {
                     Ignition.start(optimize(getConfiguration(getTestGridName(SRV_CNT))));
 
-                    fail();
+                    // Commented due to IGNITE-4473, because
+                    // IgniteClientDisconnectedException won't
+                    // be thrown, but client will reconnect.
+//                    fail();
 
-                    return false;
+                    return true;
                 }
                 catch (IgniteClientDisconnectedException e) {
                     log.info("Expected start error: " + e);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
new file mode 100644
index 0000000000000..a5d42e9a12cba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests client to be able restore connection to cluster if coordination is not available.
+ */
+public class IgniteClientRejoinTest extends GridCommonAbstractTest {
+    /** Block. */
+    private volatile boolean block;
+
+    /** Block all. */
+    private volatile boolean blockAll;
+
+    /** Coordinator. */
+    private volatile ClusterNode crd;
+
+    /** Client reconnect disabled. */
+    private boolean clientReconnectDisabled;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK", "true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.clearProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        clientReconnectDisabled = false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("client")) {
+            cfg.setCommunicationSpi(new TcpCommunicationSpi());
+
+            TcpDiscoverySpi spi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+            DiscoverySpi dspi = new DiscoverySpi();
+
+            dspi.setIpFinder(spi.getIpFinder());
+
+            cfg.setDiscoverySpi(dspi);
+
+            dspi.setJoinTimeout(60_000);
+            dspi.setClientReconnectDisabled(clientReconnectDisabled);
+
+            cfg.setClientMode(true);
+        }
+
+        // TODO: IGNITE-4833
+        cfg.setPeerClassLoadingEnabled(false);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectAfterStart() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        List<Ignite> clientNodes = new ArrayList<>();
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++)
+            clientNodes.add(startGrid("client" + i));
+
+        blockAll = true;
+
+        GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                U.sleep(5_000);
+
+                block = true;
+                blockAll = false;
+
+                System.out.println(">>> Allow with blocked coordinator.");
+
+                latch.countDown();
+
+                return null;
+            }
+        });
+
+        IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                latch.await();
+
+                U.sleep((new Random().nextInt(15) + 30) * 1000);
+
+                block = false;
+
+                System.out.println(">>> Allow coordinator.");
+
+                return null;
+            }
+        });
+
+        fut.get();
+
+        for (Ignite client : clientNodes) {
+            while (true) {
+                try {
+                    IgniteCache<Integer, Integer> cache = client.getOrCreateCache("some");
+
+                    for (int i = 0; i < 100; i++)
+                        cache.put(i, i);
+
+                    for (int i = 0; i < 100; i++)
+                        assertEquals((Integer)i, cache.get(i));
+
+                    cache.clear();
+
+                    break;
+                }
+                catch (IgniteClientDisconnectedException e) {
+                    e.reconnectFuture().get();
+                }
+            }
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnect() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        GridTestUtils.runAsync(new Callable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                latch.countDown();
+
+                Random rnd = new Random();
+
+                U.sleep((rnd.nextInt(15) + 15) * 1000);
+
+                block = false;
+
+                System.out.println(">>> ALLOW connection to coordinator.");
+
+                return true;
+            }
+        });
+
+        for (IgniteInternalFuture<Ignite> clientFut : futs) {
+            Ignite client = clientFut.get();
+
+            IgniteCache<Integer, Integer> cache = client.getOrCreateCache(client.name());
+
+            for (int i = 0; i < 100; i++)
+                cache.put(i, i);
+
+            for (int i = 0; i < 100; i++)
+                assert i == cache.get(i);
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectDisabled() throws Exception {
+        clientReconnectDisabled = true;
+
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        latch.countDown();
+
+        for (final IgniteInternalFuture<Ignite> clientFut : futs) {
+            //noinspection ThrowableNotThrown
+            GridTestUtils.assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    clientFut.get();
+
+                    return null;
+                }
+            }, IgniteCheckedException.class, null);
+        }
+
+        assertEquals(0, srv1.cluster().forClients().nodes().size());
+        assertEquals(0, srv2.cluster().forClients().nodes().size());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 3 * 60_000;
+    }
+
+    /**
+     *
+     */
+    private class TcpCommunicationSpi extends org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     *
+     */
+    private class DiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, byte[] data,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(TcpDiscoveryAbstractMessage msg, Socket sock, int res,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(msg, sock, res, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Socket openSocket(Socket sock, InetSocketAddress remAddr,
+            IgniteSpiOperationTimeoutHelper timeoutHelper) throws IOException, IgniteSpiOperationTimeoutException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            return super.openSocket(sock, remAddr, timeoutHelper);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 331b581bb7bec..0483a1ce8fbc6 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -32,6 +32,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.IgniteState;
@@ -43,6 +44,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -1788,8 +1790,7 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
         clientNodeIds.add(client.cluster().localNode().id());
 
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override
-            public boolean apply() {
+            @Override public boolean apply() {
                 return srv.cluster().nodes().size() == 2;
             }
         }, awaitTime());
@@ -1799,6 +1800,49 @@ public boolean apply() {
         assertFalse(err.get());
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testForceClientReconnect() throws Exception {
+        startServerNodes(1);
+
+        startClientNodes(1);
+
+        Ignite srv = G.ignite("server-0");
+        IgniteKernal client = (IgniteKernal)G.ignite("client-0");
+
+        UUID clientId = F.first(clientNodeIds);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        srv.events().enableLocal(EVT_NODE_JOINED);
+
+        srv.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                latch.countDown();
+
+                return false;
+            }
+        }, EVT_NODE_JOINED);
+
+        client.context().discovery().reconnect();
+
+        assert latch.await(10, TimeUnit.SECONDS);
+
+        while (true) {
+            try {
+                UUID newId = client.localNode().id();
+
+                assert !clientId.equals(newId) : clientId;
+
+                break;
+            }
+            catch (IgniteClientDisconnectedException e) {
+                e.reconnectFuture().get(10_000);
+            }
+        }
+    }
+
     /**
      * @param ignite Ignite.
      * @throws Exception If failed.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index ea8e37bada1d0..67d88e1e29eac 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.IgniteClientReconnectServicesTest;
 import org.apache.ignite.internal.IgniteClientReconnectStopTest;
 import org.apache.ignite.internal.IgniteClientReconnectStreamerTest;
+import org.apache.ignite.internal.IgniteClientRejoinTest;
 
 /**
  *
@@ -52,6 +53,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteClientReconnectServicesTest.class);
         suite.addTestSuite(IgniteClientReconnectStreamerTest.class);
         suite.addTestSuite(IgniteClientReconnectFailoverTest.class);
+        suite.addTestSuite(IgniteClientRejoinTest.class);
 
         return suite;
     }

From a2b4751f5eefd70a5a1aa26652c9671240125f78 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 17 Mar 2017 14:57:48 +0300
Subject: [PATCH 049/516] IGNITE-4473 - Client should re-try connection attempt
 in case of concurrent network failure.

(cherry picked from commit d124004)
---
 .../internal/GridKernalGatewayImpl.java       |   8 +-
 .../apache/ignite/internal/IgniteKernal.java  | 120 +++++-
 .../IgniteNeedReconnectException.java         |  40 ++
 .../discovery/GridDiscoveryManager.java       |  24 ++
 .../GridCachePartitionExchangeManager.java    |  25 +-
 .../dht/GridDhtAssignmentFetchFuture.java     |  14 +-
 .../GridDhtPartitionsExchangeFuture.java      |  48 ++-
 .../service/GridServiceProcessor.java         |  86 ++--
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 201 ++++++++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   5 +
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |   8 +
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   9 +
 .../IgniteClientReconnectCacheTest.java       |   7 +-
 .../internal/IgniteClientRejoinTest.java      | 378 ++++++++++++++++++
 .../tcp/TcpClientDiscoverySpiSelfTest.java    |  48 ++-
 .../IgniteClientReconnectTestSuite.java       |   2 +
 16 files changed, 929 insertions(+), 94 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
index fe8c580ca63ea..036954a3a280d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
@@ -44,7 +44,7 @@ public class GridKernalGatewayImpl implements GridKernalGateway, Serializable {
 
     /** */
     @GridToStringExclude
-    private IgniteFutureImpl<?> reconnectFut;
+    private volatile IgniteFutureImpl<?> reconnectFut;
 
     /** */
     private final AtomicReference<GridKernalState> state = new AtomicReference<>(GridKernalState.STOPPED);
@@ -149,6 +149,12 @@ public GridKernalGatewayImpl(String gridName) {
 
     /** {@inheritDoc} */
     @Override public GridFutureAdapter<?> onDisconnected() {
+        if (state.get() == GridKernalState.DISCONNECTED) {
+            assert reconnectFut != null;
+
+            return (GridFutureAdapter<?>)reconnectFut.internalFuture();
+        }
+
         GridFutureAdapter<?> fut = new GridFutureAdapter<>();
 
         reconnectFut = new IgniteFutureImpl<>(fut);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 8fda72fc18419..25f7884c889b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -250,6 +250,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     /** Periodic starvation check interval. */
     private static final long PERIODIC_STARVATION_CHECK_FREQ = 1000 * 30;
 
+    /** Force complete reconnect future. */
+    private static final Object STOP_RECONNECT = new Object();
+
     /** */
     @GridToStringExclude
     private GridKernalContextImpl ctx;
@@ -327,6 +330,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     @GridToStringExclude
     private final AtomicBoolean stopGuard = new AtomicBoolean();
 
+    /** */
+    private final ReconnectState reconnectState = new ReconnectState();
+
     /**
      * No-arg constructor is required by externalization.
      */
@@ -930,6 +936,8 @@ public void start(final IgniteConfiguration cfg,
             // Notify IO manager the second so further components can send and receive messages.
             ctx.io().onKernalStart();
 
+            boolean recon = false;
+
             // Callbacks.
             for (GridComponent comp : ctx) {
                 // Skip discovery manager.
@@ -940,10 +948,24 @@ public void start(final IgniteConfiguration cfg,
                 if (comp instanceof GridIoManager)
                     continue;
 
-                if (!skipDaemon(comp))
-                    comp.onKernalStart();
+                if (!skipDaemon(comp)) {
+                    try {
+                        comp.onKernalStart();
+                    }
+                    catch (IgniteNeedReconnectException e) {
+                        assert ctx.discovery().reconnectSupported();
+
+                        if (log.isDebugEnabled())
+                            log.debug("Failed to start node components on node start, will wait for reconnect: " + e);
+
+                        recon = true;
+                    }
+                }
             }
 
+            if (recon)
+                reconnectState.waitFirstReconnect();
+
             // Register MBeans.
             registerKernalMBean();
             registerLocalNodeMBean();
@@ -3274,6 +3296,8 @@ private void unguard() {
     public void onDisconnected() {
         Throwable err = null;
 
+        reconnectState.waitPreviousReconnect();
+
         GridFutureAdapter<?> reconnectFut = ctx.gateway().onDisconnected();
 
         if (reconnectFut == null) {
@@ -3282,9 +3306,18 @@ public void onDisconnected() {
             return;
         }
 
-        IgniteFuture<?> userFut = new IgniteFutureImpl<>(reconnectFut);
+        IgniteFutureImpl<?> curFut = (IgniteFutureImpl<?>)ctx.cluster().get().clientReconnectFuture();
+
+        IgniteFuture<?> userFut;
 
-        ctx.cluster().get().clientReconnectFuture(userFut);
+        // In case of previous reconnect did not finish keep reconnect future.
+        if (curFut != null && curFut.internalFuture() == reconnectFut)
+            userFut = curFut;
+        else {
+            userFut = new IgniteFutureImpl<>(reconnectFut);
+
+            ctx.cluster().get().clientReconnectFuture(userFut);
+        }
 
         ctx.disconnected(true);
 
@@ -3337,30 +3370,53 @@ public void onReconnected(final boolean clusterRestarted) {
         try {
             ctx.disconnected(false);
 
-            GridCompoundFuture<?, ?> reconnectFut = new GridCompoundFuture<>();
+            GridCompoundFuture curReconnectFut = reconnectState.curReconnectFut = new GridCompoundFuture<>();
+
+            reconnectState.reconnectDone = new GridFutureAdapter<>();
 
             for (GridComponent comp : ctx.components()) {
                 IgniteInternalFuture<?> fut = comp.onReconnected(clusterRestarted);
 
                 if (fut != null)
-                    reconnectFut.add((IgniteInternalFuture)fut);
+                    curReconnectFut.add(fut);
             }
 
-            reconnectFut.add((IgniteInternalFuture)ctx.cache().context().exchange().reconnectExchangeFuture());
+            curReconnectFut.add(ctx.cache().context().exchange().reconnectExchangeFuture());
+
+            curReconnectFut.markInitialized();
 
-            reconnectFut.markInitialized();
+            final GridFutureAdapter reconnectDone = reconnectState.reconnectDone;
 
-            reconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
+            curReconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
                 @Override public void apply(IgniteInternalFuture<?> fut) {
                     try {
-                        fut.get();
+                        Object res = fut.get();
+
+                        if (res == STOP_RECONNECT)
+                            return;
 
                         ctx.gateway().onReconnected();
+
+                        reconnectState.firstReconnectFut.onDone();
                     }
                     catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to reconnect, will stop node", e);
+                        if (!X.hasCause(e, IgniteNeedReconnectException.class,
+                            IgniteClientDisconnectedCheckedException.class)) {
+                            U.error(log, "Failed to reconnect, will stop node.", e);
+
+                            reconnectState.firstReconnectFut.onDone(e);
 
-                        close();
+                            close();
+                        }
+                        else {
+                            assert ctx.discovery().reconnectSupported();
+
+                            U.error(log, "Failed to finish reconnect, will retry [locNodeId=" + ctx.localNodeId() +
+                                ", err=" + e.getMessage() + ']');
+                        }
+                    }
+                    finally {
+                        reconnectDone.onDone();
                     }
                 }
             });
@@ -3519,6 +3575,46 @@ public void dumpDebugInfo() {
         }
     }
 
+    /**
+     *
+     */
+    private class ReconnectState {
+        /** */
+        private final GridFutureAdapter firstReconnectFut = new GridFutureAdapter();
+
+        /** */
+        private GridCompoundFuture<?, Object> curReconnectFut;
+
+        /** */
+        private GridFutureAdapter<?> reconnectDone;
+
+        /**
+         * @throws IgniteCheckedException If failed.
+         */
+        void waitFirstReconnect() throws IgniteCheckedException {
+            firstReconnectFut.get();
+        }
+
+        /**
+         *
+         */
+        void waitPreviousReconnect() {
+            if (curReconnectFut != null && !curReconnectFut.isDone()) {
+                assert reconnectDone != null;
+
+                curReconnectFut.onDone(STOP_RECONNECT);
+
+                try {
+                    reconnectDone.get();
+                }
+                catch (IgniteCheckedException ignote) {
+                    // No-op.
+                }
+            }
+
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IgniteKernal.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
new file mode 100644
index 0000000000000..61ab5762b95ac
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Indicates that node should try reconnect to cluster.
+ */
+public class IgniteNeedReconnectException extends IgniteCheckedException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * @param locNode Local node.
+     * @param cause Cause.
+     */
+    public IgniteNeedReconnectException(ClusterNode locNode, @Nullable Throwable cause) {
+        super("Local node need try to reconnect [locNodeId=" + locNode.id() + ']', cause);
+
+        assert locNode.isClient();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 9aa4db1e0042f..2ec10705bedbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -112,6 +112,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -1890,6 +1891,29 @@ public void failNode(UUID nodeId, @Nullable String warning) {
         }
     }
 
+    /**
+     * @return {@code True} if local node client and discovery SPI supports reconnect.
+     */
+    public boolean reconnectSupported() {
+        DiscoverySpi spi = getSpi();
+
+        return ctx.clientNode() && (spi instanceof TcpDiscoverySpi) &&
+            !(((TcpDiscoverySpi) spi).isClientReconnectDisabled());
+    }
+
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() {
+        assert reconnectSupported();
+
+        DiscoverySpi discoverySpi = getSpi();
+
+        ((TcpDiscoverySpi)discoverySpi).reconnect();
+    }
+
     /**
      * Updates topology version if current version is smaller than updated.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7f11dc45aacb7..92142c0731240 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -43,6 +43,7 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
@@ -447,6 +448,15 @@ else if (m instanceof GridDhtPartitionDemandMessage)
                     else
                         U.warn(log, "Still waiting for initial partition map exchange [fut=" + fut + ']');
                 }
+                catch (IgniteNeedReconnectException e) {
+                    throw e;
+                }
+                catch (Exception e) {
+                    if (fut.reconnectOnError(e))
+                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                    throw e;
+                }
             }
 
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
@@ -1690,6 +1700,12 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                         dumpedObjects++;
                                     }
                                 }
+                                catch (Exception e) {
+                                    if (exchFut.reconnectOnError(e))
+                                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                                    throw e;
+                                }
                             }
 
 
@@ -1829,7 +1845,14 @@ else if (r != null) {
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
                 }
-                catch (IgniteClientDisconnectedCheckedException e) {
+                catch (IgniteClientDisconnectedCheckedException | IgniteNeedReconnectException e) {
+                    assert cctx.discovery().reconnectSupported();
+
+                    U.warn(log,"Local node failed to complete partition map exchange due to " +
+                        "network issues, will try to reconnect to cluster", e);
+
+                    cctx.discovery().reconnect();
+
                     return;
                 }
                 catch (IgniteCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index ab8e863e90e7d..6425bc141245d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -17,15 +17,16 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.Queue;
 import java.util.UUID;
-import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
@@ -34,6 +35,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -202,8 +204,14 @@ private void requestFromNextNode() {
                         "continue to another node): " + node);
                 }
                 catch (IgniteCheckedException e) {
-                    U.error(log0, "Failed to request affinity assignment from remote node (will " +
-                        "continue to another node): " + node, e);
+                    if (ctx.discovery().reconnectSupported() && X.hasCause(e, IOException.class)) {
+                        onDone(new IgniteNeedReconnectException(ctx.localNode(), e));
+
+                        return;
+                    }
+
+                    U.warn(log0, "Failed to request affinity assignment from remote node (will " +
+                        "continue to another node): " + node);
                 }
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index e945de958a339..d4f95e5b55d05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -32,6 +33,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
@@ -39,6 +41,7 @@
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -54,7 +57,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
@@ -65,7 +67,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -506,10 +508,17 @@ public void init() throws IgniteInterruptedCheckedException {
 
             throw e;
         }
+        catch (IgniteNeedReconnectException e) {
+            onDone(e);
+        }
         catch (Throwable e) {
-            U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else {
+                U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
 
-            onDone(e);
+                onDone(e);
+            }
 
             if (e instanceof Error)
                 throw (Error)e;
@@ -1297,7 +1306,10 @@ private void onAllReceived(boolean discoThread) {
             }
         }
         catch (IgniteCheckedException e) {
-            onDone(e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else
+                onDone(e);
         }
     }
 
@@ -1314,8 +1326,15 @@ private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
         }
         catch (IgniteCheckedException e) {
             if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) {
-                log.debug("Failed to send full partition map to node, node left grid " +
-                    "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+                if (log.isDebugEnabled())
+                    log.debug("Failed to send full partition map to node, node left grid " +
+                        "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+
+                return;
+            }
+
+            if (reconnectOnError(e)) {
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
 
                 return;
             }
@@ -1641,6 +1660,12 @@ public void onNodeLeft(final ClusterNode node) {
                             }
                         }
                     }
+                    catch (Exception e) {
+                        if (reconnectOnError(e))
+                            onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+                        else
+                            throw e;
+                    }
                     finally {
                         leaveBusy();
                     }
@@ -1652,6 +1677,15 @@ public void onNodeLeft(final ClusterNode node) {
         }
     }
 
+    /**
+     * @param e Exception.
+     * @return {@code True} if local node should try reconnect in case of error.
+     */
+    public boolean reconnectOnError(Throwable e) {
+        return X.hasCause(e, IOException.class, IgniteClientDisconnectedCheckedException.class) &&
+            cctx.discovery().reconnectSupported();
+    }
+
     /** {@inheritDoc} */
     @Override public int compareTo(GridDhtPartitionsExchangeFuture fut) {
         return exchId.compareTo(fut.exchId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6bcfd65cb59e0..bd815189fb5ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1498,60 +1498,60 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
         }
     }
 
-        /**
-         * Deployment callback.
-         *
-         * @param dep Service deployment.
-         * @param topVer Topology version.
-         */
-        private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
-            // Retry forever.
-            try {
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+    /**
+     * Deployment callback.
+     *
+     * @param dep Service deployment.
+     * @param topVer Topology version.
+     */
+    private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
+        // Retry forever.
+        try {
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                // If topology version changed, reassignment will happen from topology event.
-                if (newTopVer.equals(topVer))
-                    reassign(dep, topVer);
-            }
-            catch (IgniteCheckedException e) {
-                if (!(e instanceof ClusterTopologyCheckedException))
-                    log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
+            // If topology version changed, reassignment will happen from topology event.
+            if (newTopVer.equals(topVer))
+                reassign(dep, topVer);
+        }
+        catch (IgniteCheckedException e) {
+            if (!(e instanceof ClusterTopologyCheckedException))
+                log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
 
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                if (!newTopVer.equals(topVer)) {
-                    assert newTopVer.compareTo(topVer) > 0;
+            if (!newTopVer.equals(topVer)) {
+                assert newTopVer.compareTo(topVer) > 0;
 
-                    // Reassignment will happen from topology event.
-                    return;
-                }
+                // Reassignment will happen from topology event.
+                return;
+            }
 
-                ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
-                    private IgniteUuid id = IgniteUuid.randomUuid();
+            ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
+                private IgniteUuid id = IgniteUuid.randomUuid();
 
-                    private long start = System.currentTimeMillis();
+                private long start = System.currentTimeMillis();
 
-                    @Override public IgniteUuid timeoutId() {
-                        return id;
-                    }
+                @Override public IgniteUuid timeoutId() {
+                    return id;
+                }
 
-                    @Override public long endTime() {
-                        return start + RETRY_TIMEOUT;
-                    }
+                @Override public long endTime() {
+                    return start + RETRY_TIMEOUT;
+                }
 
-                    @Override public void onTimeout() {
-                        if (!busyLock.enterBusy())
-                            return;
+                @Override public void onTimeout() {
+                    if (!busyLock.enterBusy())
+                        return;
 
-                        try {
-                            // Try again.
-                            onDeployment(dep, topVer);
-                        }
-                        finally {
-                            busyLock.leaveBusy();
-                        }
+                    try {
+                        // Try again.
+                        onDeployment(dep, topVer);
                     }
-                });
+                    finally {
+                        busyLock.leaveBusy();
+                    }
+                }
+            });
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 95e2cda4f6907..02ba56a884b33 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -129,6 +129,9 @@ class ClientImpl extends TcpDiscoveryImpl {
     /** */
     private static final Object SPI_RECONNECT_FAILED = "SPI_RECONNECT_FAILED";
 
+    /** */
+    private static final Object SPI_RECONNECT = "SPI_RECONNECT";
+
     /** Remote nodes. */
     private final ConcurrentMap<UUID, TcpDiscoveryNode> rmtNodes = new ConcurrentHashMap8<>();
 
@@ -808,6 +811,11 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
             log);
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        msgWorker.addMessage(SPI_RECONNECT);
+    }
+
     /** {@inheritDoc} */
     @Override public void brakeConnection() {
         SocketStream sockStream = msgWorker.currSock;
@@ -879,9 +887,12 @@ private class SocketReader extends IgniteSpiThread {
         /** */
         private UUID rmtNodeId;
 
+        /** */
+        private CountDownLatch stopReadLatch;
+
         /**
          */
-        protected SocketReader() {
+        SocketReader() {
             super(spi.ignite().name(), "tcp-client-disco-sock-reader", log);
         }
 
@@ -889,7 +900,7 @@ protected SocketReader() {
          * @param sockStream Socket.
          * @param rmtNodeId Rmt node id.
          */
-        public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
+        void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             synchronized (mux) {
                 this.sockStream = sockStream;
 
@@ -899,6 +910,31 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             }
         }
 
+        /**
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceStopRead() throws InterruptedException {
+            CountDownLatch stopReadLatch;
+
+            synchronized (mux) {
+                SocketStream stream = sockStream;
+
+                if (stream == null)
+                    return;
+
+                this.stopReadLatch = stopReadLatch = new CountDownLatch(1);
+
+                U.closeQuiet(stream.socket());
+
+                this.sockStream = null;
+                this.rmtNodeId = null;
+
+                mux.notifyAll();
+            }
+
+            stopReadLatch.await();
+        }
+
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException {
             while (!isInterrupted()) {
@@ -906,6 +942,12 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
                 UUID rmtNodeId;
 
                 synchronized (mux) {
+                    if (stopReadLatch != null) {
+                        stopReadLatch.countDown();
+
+                        stopReadLatch = null;
+                    }
+
                     if (this.sockStream == null) {
                         mux.wait();
 
@@ -1007,18 +1049,21 @@ private class SocketWriter extends IgniteSpiThread {
         private final Queue<TcpDiscoveryAbstractMessage> queue = new ArrayDeque<>();
 
         /** */
-        private final long socketTimeout;
+        private final long sockTimeout;
 
         /** */
         private TcpDiscoveryAbstractMessage unackedMsg;
 
+        /** */
+        private CountDownLatch forceLeaveLatch;
+
         /**
          *
          */
-        protected SocketWriter() {
+        SocketWriter() {
             super(spi.ignite().name(), "tcp-client-disco-sock-writer", log);
 
-            socketTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
+            sockTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
                 spi.getSocketTimeout();
         }
 
@@ -1033,6 +1078,29 @@ private void sendMessage(TcpDiscoveryAbstractMessage msg) {
             }
         }
 
+        /**
+         * Sends {@link TcpDiscoveryNodeLeftMessage} and closes socket.
+         *
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceLeave() throws InterruptedException {
+            CountDownLatch forceLeaveLatch;
+
+            synchronized (mux) {
+                // If writer was stopped.
+                if (sock == null)
+                    return;
+
+                this.forceLeaveLatch = forceLeaveLatch = new CountDownLatch(1);
+
+                unackedMsg = null;
+
+                mux.notifyAll();
+            }
+
+            forceLeaveLatch.await();
+        }
+
         /**
          * @param sock Socket.
          * @param clientAck {@code True} is server supports client message acknowlede.
@@ -1089,13 +1157,41 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                         continue;
                     }
 
-                    msg = queue.poll();
+                    if (forceLeaveLatch != null) {
+                        msg = new TcpDiscoveryNodeLeftMessage(getLocalNodeId());
 
-                    if (msg == null) {
-                        mux.wait();
+                        msg.client(true);
+
+                        try {
+                            spi.writeToSocket(
+                                sock,
+                                msg,
+                                sockTimeout);
+                        }
+                        catch (IOException | IgniteCheckedException e) {
+                            if (log.isDebugEnabled()) {
+                                log.debug("Failed to send TcpDiscoveryNodeLeftMessage on force leave [msg=" + msg +
+                                    ", err=" + e.getMessage() + ']');
+                            }
+                        }
+
+                        U.closeQuiet(sock);
+
+                        this.sock = null;
+
+                        clear();
 
                         continue;
                     }
+                    else {
+                        msg = queue.poll();
+
+                        if (msg == null) {
+                            mux.wait();
+
+                            continue;
+                        }
+                    }
                 }
 
                 for (IgniteInClosure<TcpDiscoveryAbstractMessage> msgLsnr : spi.sndMsgLsnrs)
@@ -1115,7 +1211,7 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     spi.writeToSocket(
                         sock,
                         msg,
-                        socketTimeout);
+                        sockTimeout);
 
                     msg = null;
 
@@ -1165,10 +1261,30 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     synchronized (mux) {
                         if (sock == this.sock)
                             this.sock = null; // Connection has dead.
+
+                        clear();
                     }
                 }
             }
         }
+
+        /**
+         *
+         */
+        private void clear() {
+            assert Thread.holdsLock(mux);
+
+            queue.clear();
+            unackedMsg = null;
+
+            CountDownLatch forceLeaveLatch = this.forceLeaveLatch;
+
+            if (forceLeaveLatch != null) {
+                this.forceLeaveLatch = null;
+
+                forceLeaveLatch.countDown();
+            }
+        }
     }
 
     /**
@@ -1413,6 +1529,38 @@ else if (msg == SPI_STOP) {
                         else
                             leaveLatch.countDown();
                     }
+                    else if (msg == SPI_RECONNECT) {
+                        if (state == CONNECTED) {
+                            if (reconnector != null) {
+                                reconnector.cancel();
+                                reconnector.join();
+
+                                reconnector = null;
+                            }
+
+                            sockWriter.forceLeave();
+                            sockReader.forceStopRead();
+
+                            currSock = null;
+
+                            queue.clear();
+
+                            onDisconnected();
+
+                            notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
+
+                            UUID newId = UUID.randomUUID();
+
+                            U.quietAndWarn(log, "Local node will try to reconnect to cluster with new id due " +
+                                "to network problems [newId=" + newId +
+                                ", prevId=" + locNode.id() +
+                                ", locNode=" + locNode+ ']');
+
+                            locNode.onClientDisconnected(newId);
+
+                            tryJoin();
+                        }
+                    }
                     else if (msg instanceof TcpDiscoveryNodeFailedMessage &&
                         ((TcpDiscoveryNodeFailedMessage)msg).failedNodeId().equals(locNode.id())) {
                         TcpDiscoveryNodeFailedMessage msg0 = (TcpDiscoveryNodeFailedMessage)msg;
@@ -1495,20 +1643,7 @@ else if (msg == SPI_RECONNECT_FAILED) {
                                         ", failMsg=" + forceFailMsg + ']');
                                 }
 
-                                state = DISCONNECTED;
-
-                                nodeAdded = false;
-
-                                IgniteClientDisconnectedCheckedException err =
-                                    new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
-                                    "client node disconnected.");
-
-                                for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
-                                    GridFutureAdapter<Boolean> fut = e.getValue();
-
-                                    if (pingFuts.remove(e.getKey(), fut))
-                                        fut.onDone(err);
-                                }
+                                onDisconnected();
 
                                 notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
                             }
@@ -1603,6 +1738,26 @@ else if (discoMsg instanceof TcpDiscoveryCheckFailedMessage)
             }
         }
 
+        /**
+         *
+         */
+        private void onDisconnected() {
+            state = DISCONNECTED;
+
+            nodeAdded = false;
+
+            IgniteClientDisconnectedCheckedException err =
+                new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
+                    "client node disconnected.");
+
+            for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
+                GridFutureAdapter<Boolean> fut = e.getValue();
+
+                if (pingFuts.remove(e.getKey(), fut))
+                    fut.onDone(err);
+            }
+        }
+
         /**
          * @throws InterruptedException If interrupted.
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 4600be094c840..afd1c2ba7fc66 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1609,6 +1609,11 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
         throw new UnsupportedOperationException();
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        throw new UnsupportedOperationException("Reconnect is not supported for server.");
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteSpiThread workerThread() {
         return msgWorker;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index f199c20e17edb..84c2ff28afef0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -258,6 +259,13 @@ protected static String threadStatus(Thread t) {
         return t.isAlive() ? "alive" : "dead";
     }
 
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public abstract void reconnect() throws IgniteSpiException;
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      * <p>
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 00ae97d1e302c..a2a47feb31486 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1926,6 +1926,15 @@ boolean isSslEnabled() {
         return ignite().configuration().getSslContextFactory() != null;
     }
 
+    /**
+     * Force reconnect to cluster.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() throws IgniteSpiException {
+        impl.reconnect();
+    }
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 0f0165b0f04f9..6cdf465978907 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -700,9 +700,12 @@ public void testReconnectInitialExchangeInProgress() throws Exception {
                 try {
                     Ignition.start(optimize(getConfiguration(getTestGridName(SRV_CNT))));
 
-                    fail();
+                    // Commented due to IGNITE-4473, because
+                    // IgniteClientDisconnectedException won't
+                    // be thrown, but client will reconnect.
+//                    fail();
 
-                    return false;
+                    return true;
                 }
                 catch (IgniteClientDisconnectedException e) {
                     log.info("Expected start error: " + e);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
new file mode 100644
index 0000000000000..a5d42e9a12cba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests client to be able restore connection to cluster if coordination is not available.
+ */
+public class IgniteClientRejoinTest extends GridCommonAbstractTest {
+    /** Block. */
+    private volatile boolean block;
+
+    /** Block all. */
+    private volatile boolean blockAll;
+
+    /** Coordinator. */
+    private volatile ClusterNode crd;
+
+    /** Client reconnect disabled. */
+    private boolean clientReconnectDisabled;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK", "true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.clearProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        clientReconnectDisabled = false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("client")) {
+            cfg.setCommunicationSpi(new TcpCommunicationSpi());
+
+            TcpDiscoverySpi spi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+            DiscoverySpi dspi = new DiscoverySpi();
+
+            dspi.setIpFinder(spi.getIpFinder());
+
+            cfg.setDiscoverySpi(dspi);
+
+            dspi.setJoinTimeout(60_000);
+            dspi.setClientReconnectDisabled(clientReconnectDisabled);
+
+            cfg.setClientMode(true);
+        }
+
+        // TODO: IGNITE-4833
+        cfg.setPeerClassLoadingEnabled(false);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectAfterStart() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        List<Ignite> clientNodes = new ArrayList<>();
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++)
+            clientNodes.add(startGrid("client" + i));
+
+        blockAll = true;
+
+        GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                U.sleep(5_000);
+
+                block = true;
+                blockAll = false;
+
+                System.out.println(">>> Allow with blocked coordinator.");
+
+                latch.countDown();
+
+                return null;
+            }
+        });
+
+        IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                latch.await();
+
+                U.sleep((new Random().nextInt(15) + 30) * 1000);
+
+                block = false;
+
+                System.out.println(">>> Allow coordinator.");
+
+                return null;
+            }
+        });
+
+        fut.get();
+
+        for (Ignite client : clientNodes) {
+            while (true) {
+                try {
+                    IgniteCache<Integer, Integer> cache = client.getOrCreateCache("some");
+
+                    for (int i = 0; i < 100; i++)
+                        cache.put(i, i);
+
+                    for (int i = 0; i < 100; i++)
+                        assertEquals((Integer)i, cache.get(i));
+
+                    cache.clear();
+
+                    break;
+                }
+                catch (IgniteClientDisconnectedException e) {
+                    e.reconnectFuture().get();
+                }
+            }
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnect() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        GridTestUtils.runAsync(new Callable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                latch.countDown();
+
+                Random rnd = new Random();
+
+                U.sleep((rnd.nextInt(15) + 15) * 1000);
+
+                block = false;
+
+                System.out.println(">>> ALLOW connection to coordinator.");
+
+                return true;
+            }
+        });
+
+        for (IgniteInternalFuture<Ignite> clientFut : futs) {
+            Ignite client = clientFut.get();
+
+            IgniteCache<Integer, Integer> cache = client.getOrCreateCache(client.name());
+
+            for (int i = 0; i < 100; i++)
+                cache.put(i, i);
+
+            for (int i = 0; i < 100; i++)
+                assert i == cache.get(i);
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectDisabled() throws Exception {
+        clientReconnectDisabled = true;
+
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        latch.countDown();
+
+        for (final IgniteInternalFuture<Ignite> clientFut : futs) {
+            //noinspection ThrowableNotThrown
+            GridTestUtils.assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    clientFut.get();
+
+                    return null;
+                }
+            }, IgniteCheckedException.class, null);
+        }
+
+        assertEquals(0, srv1.cluster().forClients().nodes().size());
+        assertEquals(0, srv2.cluster().forClients().nodes().size());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 3 * 60_000;
+    }
+
+    /**
+     *
+     */
+    private class TcpCommunicationSpi extends org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     *
+     */
+    private class DiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, byte[] data,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(TcpDiscoveryAbstractMessage msg, Socket sock, int res,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(msg, sock, res, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Socket openSocket(Socket sock, InetSocketAddress remAddr,
+            IgniteSpiOperationTimeoutHelper timeoutHelper) throws IOException, IgniteSpiOperationTimeoutException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            return super.openSocket(sock, remAddr, timeoutHelper);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 331b581bb7bec..0483a1ce8fbc6 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -32,6 +32,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.IgniteState;
@@ -43,6 +44,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -1788,8 +1790,7 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
         clientNodeIds.add(client.cluster().localNode().id());
 
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override
-            public boolean apply() {
+            @Override public boolean apply() {
                 return srv.cluster().nodes().size() == 2;
             }
         }, awaitTime());
@@ -1799,6 +1800,49 @@ public boolean apply() {
         assertFalse(err.get());
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testForceClientReconnect() throws Exception {
+        startServerNodes(1);
+
+        startClientNodes(1);
+
+        Ignite srv = G.ignite("server-0");
+        IgniteKernal client = (IgniteKernal)G.ignite("client-0");
+
+        UUID clientId = F.first(clientNodeIds);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        srv.events().enableLocal(EVT_NODE_JOINED);
+
+        srv.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                latch.countDown();
+
+                return false;
+            }
+        }, EVT_NODE_JOINED);
+
+        client.context().discovery().reconnect();
+
+        assert latch.await(10, TimeUnit.SECONDS);
+
+        while (true) {
+            try {
+                UUID newId = client.localNode().id();
+
+                assert !clientId.equals(newId) : clientId;
+
+                break;
+            }
+            catch (IgniteClientDisconnectedException e) {
+                e.reconnectFuture().get(10_000);
+            }
+        }
+    }
+
     /**
      * @param ignite Ignite.
      * @throws Exception If failed.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index ea8e37bada1d0..67d88e1e29eac 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.IgniteClientReconnectServicesTest;
 import org.apache.ignite.internal.IgniteClientReconnectStopTest;
 import org.apache.ignite.internal.IgniteClientReconnectStreamerTest;
+import org.apache.ignite.internal.IgniteClientRejoinTest;
 
 /**
  *
@@ -52,6 +53,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteClientReconnectServicesTest.class);
         suite.addTestSuite(IgniteClientReconnectStreamerTest.class);
         suite.addTestSuite(IgniteClientReconnectFailoverTest.class);
+        suite.addTestSuite(IgniteClientRejoinTest.class);
 
         return suite;
     }

From 946419314b567c604a15ae4f9658d89bc350127b Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 17 Mar 2017 14:57:48 +0300
Subject: [PATCH 050/516] IGNITE-4473 - Client should re-try connection attempt
 in case of concurrent network failure.

(cherry picked from commit d124004)
---
 .../internal/GridKernalGatewayImpl.java       |   8 +-
 .../apache/ignite/internal/IgniteKernal.java  | 120 +++++-
 .../IgniteNeedReconnectException.java         |  40 ++
 .../discovery/GridDiscoveryManager.java       |  24 ++
 .../GridCachePartitionExchangeManager.java    |  25 +-
 .../dht/GridDhtAssignmentFetchFuture.java     |  14 +-
 .../GridDhtPartitionsExchangeFuture.java      |  48 ++-
 .../service/GridServiceProcessor.java         |  86 ++--
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 201 ++++++++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   5 +
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |   8 +
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   9 +
 .../IgniteClientReconnectCacheTest.java       |   7 +-
 .../internal/IgniteClientRejoinTest.java      | 378 ++++++++++++++++++
 .../tcp/TcpClientDiscoverySpiSelfTest.java    |  48 ++-
 .../IgniteClientReconnectTestSuite.java       |   2 +
 16 files changed, 929 insertions(+), 94 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
index fe8c580ca63ea..036954a3a280d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalGatewayImpl.java
@@ -44,7 +44,7 @@ public class GridKernalGatewayImpl implements GridKernalGateway, Serializable {
 
     /** */
     @GridToStringExclude
-    private IgniteFutureImpl<?> reconnectFut;
+    private volatile IgniteFutureImpl<?> reconnectFut;
 
     /** */
     private final AtomicReference<GridKernalState> state = new AtomicReference<>(GridKernalState.STOPPED);
@@ -149,6 +149,12 @@ public GridKernalGatewayImpl(String gridName) {
 
     /** {@inheritDoc} */
     @Override public GridFutureAdapter<?> onDisconnected() {
+        if (state.get() == GridKernalState.DISCONNECTED) {
+            assert reconnectFut != null;
+
+            return (GridFutureAdapter<?>)reconnectFut.internalFuture();
+        }
+
         GridFutureAdapter<?> fut = new GridFutureAdapter<>();
 
         reconnectFut = new IgniteFutureImpl<>(fut);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 4972d1f306e2e..063fe255af95f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -250,6 +250,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     /** Periodic starvation check interval. */
     private static final long PERIODIC_STARVATION_CHECK_FREQ = 1000 * 30;
 
+    /** Force complete reconnect future. */
+    private static final Object STOP_RECONNECT = new Object();
+
     /** */
     @GridToStringExclude
     private GridKernalContextImpl ctx;
@@ -327,6 +330,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     @GridToStringExclude
     private final AtomicBoolean stopGuard = new AtomicBoolean();
 
+    /** */
+    private final ReconnectState reconnectState = new ReconnectState();
+
     /**
      * No-arg constructor is required by externalization.
      */
@@ -936,6 +942,8 @@ public void start(
             // Notify IO manager the second so further components can send and receive messages.
             ctx.io().onKernalStart();
 
+            boolean recon = false;
+
             // Callbacks.
             for (GridComponent comp : ctx) {
                 // Skip discovery manager.
@@ -946,10 +954,24 @@ public void start(
                 if (comp instanceof GridIoManager)
                     continue;
 
-                if (!skipDaemon(comp))
-                    comp.onKernalStart();
+                if (!skipDaemon(comp)) {
+                    try {
+                        comp.onKernalStart();
+                    }
+                    catch (IgniteNeedReconnectException e) {
+                        assert ctx.discovery().reconnectSupported();
+
+                        if (log.isDebugEnabled())
+                            log.debug("Failed to start node components on node start, will wait for reconnect: " + e);
+
+                        recon = true;
+                    }
+                }
             }
 
+            if (recon)
+                reconnectState.waitFirstReconnect();
+
             // Register MBeans.
             registerKernalMBean();
             registerLocalNodeMBean();
@@ -3309,6 +3331,8 @@ private void unguard() {
     public void onDisconnected() {
         Throwable err = null;
 
+        reconnectState.waitPreviousReconnect();
+
         GridFutureAdapter<?> reconnectFut = ctx.gateway().onDisconnected();
 
         if (reconnectFut == null) {
@@ -3317,9 +3341,18 @@ public void onDisconnected() {
             return;
         }
 
-        IgniteFuture<?> userFut = new IgniteFutureImpl<>(reconnectFut);
+        IgniteFutureImpl<?> curFut = (IgniteFutureImpl<?>)ctx.cluster().get().clientReconnectFuture();
+
+        IgniteFuture<?> userFut;
 
-        ctx.cluster().get().clientReconnectFuture(userFut);
+        // In case of previous reconnect did not finish keep reconnect future.
+        if (curFut != null && curFut.internalFuture() == reconnectFut)
+            userFut = curFut;
+        else {
+            userFut = new IgniteFutureImpl<>(reconnectFut);
+
+            ctx.cluster().get().clientReconnectFuture(userFut);
+        }
 
         ctx.disconnected(true);
 
@@ -3372,30 +3405,53 @@ public void onReconnected(final boolean clusterRestarted) {
         try {
             ctx.disconnected(false);
 
-            GridCompoundFuture<?, ?> reconnectFut = new GridCompoundFuture<>();
+            GridCompoundFuture curReconnectFut = reconnectState.curReconnectFut = new GridCompoundFuture<>();
+
+            reconnectState.reconnectDone = new GridFutureAdapter<>();
 
             for (GridComponent comp : ctx.components()) {
                 IgniteInternalFuture<?> fut = comp.onReconnected(clusterRestarted);
 
                 if (fut != null)
-                    reconnectFut.add((IgniteInternalFuture)fut);
+                    curReconnectFut.add(fut);
             }
 
-            reconnectFut.add((IgniteInternalFuture)ctx.cache().context().exchange().reconnectExchangeFuture());
+            curReconnectFut.add(ctx.cache().context().exchange().reconnectExchangeFuture());
+
+            curReconnectFut.markInitialized();
 
-            reconnectFut.markInitialized();
+            final GridFutureAdapter reconnectDone = reconnectState.reconnectDone;
 
-            reconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
+            curReconnectFut.listen(new CI1<IgniteInternalFuture<?>>() {
                 @Override public void apply(IgniteInternalFuture<?> fut) {
                     try {
-                        fut.get();
+                        Object res = fut.get();
+
+                        if (res == STOP_RECONNECT)
+                            return;
 
                         ctx.gateway().onReconnected();
+
+                        reconnectState.firstReconnectFut.onDone();
                     }
                     catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to reconnect, will stop node", e);
+                        if (!X.hasCause(e, IgniteNeedReconnectException.class,
+                            IgniteClientDisconnectedCheckedException.class)) {
+                            U.error(log, "Failed to reconnect, will stop node.", e);
+
+                            reconnectState.firstReconnectFut.onDone(e);
 
-                        close();
+                            close();
+                        }
+                        else {
+                            assert ctx.discovery().reconnectSupported();
+
+                            U.error(log, "Failed to finish reconnect, will retry [locNodeId=" + ctx.localNodeId() +
+                                ", err=" + e.getMessage() + ']');
+                        }
+                    }
+                    finally {
+                        reconnectDone.onDone();
                     }
                 }
             });
@@ -3574,6 +3630,46 @@ public IgniteInternalFuture sendIoTest(List<ClusterNode> nodes, byte[] payload,
         return ctx.io().sendIoTest(nodes, payload, procFromNioThread);
     }
 
+    /**
+     *
+     */
+    private class ReconnectState {
+        /** */
+        private final GridFutureAdapter firstReconnectFut = new GridFutureAdapter();
+
+        /** */
+        private GridCompoundFuture<?, Object> curReconnectFut;
+
+        /** */
+        private GridFutureAdapter<?> reconnectDone;
+
+        /**
+         * @throws IgniteCheckedException If failed.
+         */
+        void waitFirstReconnect() throws IgniteCheckedException {
+            firstReconnectFut.get();
+        }
+
+        /**
+         *
+         */
+        void waitPreviousReconnect() {
+            if (curReconnectFut != null && !curReconnectFut.isDone()) {
+                assert reconnectDone != null;
+
+                curReconnectFut.onDone(STOP_RECONNECT);
+
+                try {
+                    reconnectDone.get();
+                }
+                catch (IgniteCheckedException ignote) {
+                    // No-op.
+                }
+            }
+
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IgniteKernal.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
new file mode 100644
index 0000000000000..61ab5762b95ac
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Indicates that node should try reconnect to cluster.
+ */
+public class IgniteNeedReconnectException extends IgniteCheckedException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * @param locNode Local node.
+     * @param cause Cause.
+     */
+    public IgniteNeedReconnectException(ClusterNode locNode, @Nullable Throwable cause) {
+        super("Local node need try to reconnect [locNodeId=" + locNode.id() + ']', cause);
+
+        assert locNode.isClient();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 9aa4db1e0042f..2ec10705bedbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -112,6 +112,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -1890,6 +1891,29 @@ public void failNode(UUID nodeId, @Nullable String warning) {
         }
     }
 
+    /**
+     * @return {@code True} if local node client and discovery SPI supports reconnect.
+     */
+    public boolean reconnectSupported() {
+        DiscoverySpi spi = getSpi();
+
+        return ctx.clientNode() && (spi instanceof TcpDiscoverySpi) &&
+            !(((TcpDiscoverySpi) spi).isClientReconnectDisabled());
+    }
+
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() {
+        assert reconnectSupported();
+
+        DiscoverySpi discoverySpi = getSpi();
+
+        ((TcpDiscoverySpi)discoverySpi).reconnect();
+    }
+
     /**
      * Updates topology version if current version is smaller than updated.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7cf75fee86be3..6f2c62608670f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -43,6 +43,7 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
@@ -448,6 +449,15 @@ else if (m instanceof GridDhtPartitionDemandMessage)
                     else
                         U.warn(log, "Still waiting for initial partition map exchange [fut=" + fut + ']');
                 }
+                catch (IgniteNeedReconnectException e) {
+                    throw e;
+                }
+                catch (Exception e) {
+                    if (fut.reconnectOnError(e))
+                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                    throw e;
+                }
             }
 
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
@@ -1697,6 +1707,12 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                         dumpedObjects++;
                                     }
                                 }
+                                catch (Exception e) {
+                                    if (exchFut.reconnectOnError(e))
+                                        throw new IgniteNeedReconnectException(cctx.localNode(), e);
+
+                                    throw e;
+                                }
                             }
 
 
@@ -1836,7 +1852,14 @@ else if (r != null) {
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
                 }
-                catch (IgniteClientDisconnectedCheckedException e) {
+                catch (IgniteClientDisconnectedCheckedException | IgniteNeedReconnectException e) {
+                    assert cctx.discovery().reconnectSupported();
+
+                    U.warn(log,"Local node failed to complete partition map exchange due to " +
+                        "network issues, will try to reconnect to cluster", e);
+
+                    cctx.discovery().reconnect();
+
                     return;
                 }
                 catch (IgniteCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index ab8e863e90e7d..6425bc141245d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -17,15 +17,16 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.Queue;
 import java.util.UUID;
-import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
@@ -34,6 +35,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -202,8 +204,14 @@ private void requestFromNextNode() {
                         "continue to another node): " + node);
                 }
                 catch (IgniteCheckedException e) {
-                    U.error(log0, "Failed to request affinity assignment from remote node (will " +
-                        "continue to another node): " + node, e);
+                    if (ctx.discovery().reconnectSupported() && X.hasCause(e, IOException.class)) {
+                        onDone(new IgniteNeedReconnectException(ctx.localNode(), e));
+
+                        return;
+                    }
+
+                    U.warn(log0, "Failed to request affinity assignment from remote node (will " +
+                        "continue to another node): " + node);
                 }
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index e945de958a339..d4f95e5b55d05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -32,6 +33,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
@@ -39,6 +41,7 @@
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -54,7 +57,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
@@ -65,7 +67,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -506,10 +508,17 @@ public void init() throws IgniteInterruptedCheckedException {
 
             throw e;
         }
+        catch (IgniteNeedReconnectException e) {
+            onDone(e);
+        }
         catch (Throwable e) {
-            U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else {
+                U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
 
-            onDone(e);
+                onDone(e);
+            }
 
             if (e instanceof Error)
                 throw (Error)e;
@@ -1297,7 +1306,10 @@ private void onAllReceived(boolean discoThread) {
             }
         }
         catch (IgniteCheckedException e) {
-            onDone(e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else
+                onDone(e);
         }
     }
 
@@ -1314,8 +1326,15 @@ private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
         }
         catch (IgniteCheckedException e) {
             if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) {
-                log.debug("Failed to send full partition map to node, node left grid " +
-                    "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+                if (log.isDebugEnabled())
+                    log.debug("Failed to send full partition map to node, node left grid " +
+                        "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']');
+
+                return;
+            }
+
+            if (reconnectOnError(e)) {
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
 
                 return;
             }
@@ -1641,6 +1660,12 @@ public void onNodeLeft(final ClusterNode node) {
                             }
                         }
                     }
+                    catch (Exception e) {
+                        if (reconnectOnError(e))
+                            onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+                        else
+                            throw e;
+                    }
                     finally {
                         leaveBusy();
                     }
@@ -1652,6 +1677,15 @@ public void onNodeLeft(final ClusterNode node) {
         }
     }
 
+    /**
+     * @param e Exception.
+     * @return {@code True} if local node should try reconnect in case of error.
+     */
+    public boolean reconnectOnError(Throwable e) {
+        return X.hasCause(e, IOException.class, IgniteClientDisconnectedCheckedException.class) &&
+            cctx.discovery().reconnectSupported();
+    }
+
     /** {@inheritDoc} */
     @Override public int compareTo(GridDhtPartitionsExchangeFuture fut) {
         return exchId.compareTo(fut.exchId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index f7f370ee56f56..730c6a4aad390 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1498,60 +1498,60 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
         }
     }
 
-        /**
-         * Deployment callback.
-         *
-         * @param dep Service deployment.
-         * @param topVer Topology version.
-         */
-        private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
-            // Retry forever.
-            try {
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+    /**
+     * Deployment callback.
+     *
+     * @param dep Service deployment.
+     * @param topVer Topology version.
+     */
+    private void onDeployment(final GridServiceDeployment dep, final AffinityTopologyVersion topVer) {
+        // Retry forever.
+        try {
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                // If topology version changed, reassignment will happen from topology event.
-                if (newTopVer.equals(topVer))
-                    reassign(dep, topVer);
-            }
-            catch (IgniteCheckedException e) {
-                if (!(e instanceof ClusterTopologyCheckedException))
-                    log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
+            // If topology version changed, reassignment will happen from topology event.
+            if (newTopVer.equals(topVer))
+                reassign(dep, topVer);
+        }
+        catch (IgniteCheckedException e) {
+            if (!(e instanceof ClusterTopologyCheckedException))
+                log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
 
-                AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
+            AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-                if (!newTopVer.equals(topVer)) {
-                    assert newTopVer.compareTo(topVer) > 0;
+            if (!newTopVer.equals(topVer)) {
+                assert newTopVer.compareTo(topVer) > 0;
 
-                    // Reassignment will happen from topology event.
-                    return;
-                }
+                // Reassignment will happen from topology event.
+                return;
+            }
 
-                ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
-                    private IgniteUuid id = IgniteUuid.randomUuid();
+            ctx.timeout().addTimeoutObject(new GridTimeoutObject() {
+                private IgniteUuid id = IgniteUuid.randomUuid();
 
-                    private long start = System.currentTimeMillis();
+                private long start = System.currentTimeMillis();
 
-                    @Override public IgniteUuid timeoutId() {
-                        return id;
-                    }
+                @Override public IgniteUuid timeoutId() {
+                    return id;
+                }
 
-                    @Override public long endTime() {
-                        return start + RETRY_TIMEOUT;
-                    }
+                @Override public long endTime() {
+                    return start + RETRY_TIMEOUT;
+                }
 
-                    @Override public void onTimeout() {
-                        if (!busyLock.enterBusy())
-                            return;
+                @Override public void onTimeout() {
+                    if (!busyLock.enterBusy())
+                        return;
 
-                        try {
-                            // Try again.
-                            onDeployment(dep, topVer);
-                        }
-                        finally {
-                            busyLock.leaveBusy();
-                        }
+                    try {
+                        // Try again.
+                        onDeployment(dep, topVer);
                     }
-                });
+                    finally {
+                        busyLock.leaveBusy();
+                    }
+                }
+            });
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 95e2cda4f6907..02ba56a884b33 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -129,6 +129,9 @@ class ClientImpl extends TcpDiscoveryImpl {
     /** */
     private static final Object SPI_RECONNECT_FAILED = "SPI_RECONNECT_FAILED";
 
+    /** */
+    private static final Object SPI_RECONNECT = "SPI_RECONNECT";
+
     /** Remote nodes. */
     private final ConcurrentMap<UUID, TcpDiscoveryNode> rmtNodes = new ConcurrentHashMap8<>();
 
@@ -808,6 +811,11 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
             log);
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        msgWorker.addMessage(SPI_RECONNECT);
+    }
+
     /** {@inheritDoc} */
     @Override public void brakeConnection() {
         SocketStream sockStream = msgWorker.currSock;
@@ -879,9 +887,12 @@ private class SocketReader extends IgniteSpiThread {
         /** */
         private UUID rmtNodeId;
 
+        /** */
+        private CountDownLatch stopReadLatch;
+
         /**
          */
-        protected SocketReader() {
+        SocketReader() {
             super(spi.ignite().name(), "tcp-client-disco-sock-reader", log);
         }
 
@@ -889,7 +900,7 @@ protected SocketReader() {
          * @param sockStream Socket.
          * @param rmtNodeId Rmt node id.
          */
-        public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
+        void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             synchronized (mux) {
                 this.sockStream = sockStream;
 
@@ -899,6 +910,31 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
             }
         }
 
+        /**
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceStopRead() throws InterruptedException {
+            CountDownLatch stopReadLatch;
+
+            synchronized (mux) {
+                SocketStream stream = sockStream;
+
+                if (stream == null)
+                    return;
+
+                this.stopReadLatch = stopReadLatch = new CountDownLatch(1);
+
+                U.closeQuiet(stream.socket());
+
+                this.sockStream = null;
+                this.rmtNodeId = null;
+
+                mux.notifyAll();
+            }
+
+            stopReadLatch.await();
+        }
+
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException {
             while (!isInterrupted()) {
@@ -906,6 +942,12 @@ public void setSocket(SocketStream sockStream, UUID rmtNodeId) {
                 UUID rmtNodeId;
 
                 synchronized (mux) {
+                    if (stopReadLatch != null) {
+                        stopReadLatch.countDown();
+
+                        stopReadLatch = null;
+                    }
+
                     if (this.sockStream == null) {
                         mux.wait();
 
@@ -1007,18 +1049,21 @@ private class SocketWriter extends IgniteSpiThread {
         private final Queue<TcpDiscoveryAbstractMessage> queue = new ArrayDeque<>();
 
         /** */
-        private final long socketTimeout;
+        private final long sockTimeout;
 
         /** */
         private TcpDiscoveryAbstractMessage unackedMsg;
 
+        /** */
+        private CountDownLatch forceLeaveLatch;
+
         /**
          *
          */
-        protected SocketWriter() {
+        SocketWriter() {
             super(spi.ignite().name(), "tcp-client-disco-sock-writer", log);
 
-            socketTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
+            sockTimeout = spi.failureDetectionTimeoutEnabled() ? spi.failureDetectionTimeout() :
                 spi.getSocketTimeout();
         }
 
@@ -1033,6 +1078,29 @@ private void sendMessage(TcpDiscoveryAbstractMessage msg) {
             }
         }
 
+        /**
+         * Sends {@link TcpDiscoveryNodeLeftMessage} and closes socket.
+         *
+         * @throws InterruptedException If interrupted.
+         */
+        private void forceLeave() throws InterruptedException {
+            CountDownLatch forceLeaveLatch;
+
+            synchronized (mux) {
+                // If writer was stopped.
+                if (sock == null)
+                    return;
+
+                this.forceLeaveLatch = forceLeaveLatch = new CountDownLatch(1);
+
+                unackedMsg = null;
+
+                mux.notifyAll();
+            }
+
+            forceLeaveLatch.await();
+        }
+
         /**
          * @param sock Socket.
          * @param clientAck {@code True} is server supports client message acknowlede.
@@ -1089,13 +1157,41 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                         continue;
                     }
 
-                    msg = queue.poll();
+                    if (forceLeaveLatch != null) {
+                        msg = new TcpDiscoveryNodeLeftMessage(getLocalNodeId());
 
-                    if (msg == null) {
-                        mux.wait();
+                        msg.client(true);
+
+                        try {
+                            spi.writeToSocket(
+                                sock,
+                                msg,
+                                sockTimeout);
+                        }
+                        catch (IOException | IgniteCheckedException e) {
+                            if (log.isDebugEnabled()) {
+                                log.debug("Failed to send TcpDiscoveryNodeLeftMessage on force leave [msg=" + msg +
+                                    ", err=" + e.getMessage() + ']');
+                            }
+                        }
+
+                        U.closeQuiet(sock);
+
+                        this.sock = null;
+
+                        clear();
 
                         continue;
                     }
+                    else {
+                        msg = queue.poll();
+
+                        if (msg == null) {
+                            mux.wait();
+
+                            continue;
+                        }
+                    }
                 }
 
                 for (IgniteInClosure<TcpDiscoveryAbstractMessage> msgLsnr : spi.sndMsgLsnrs)
@@ -1115,7 +1211,7 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     spi.writeToSocket(
                         sock,
                         msg,
-                        socketTimeout);
+                        sockTimeout);
 
                     msg = null;
 
@@ -1165,10 +1261,30 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                     synchronized (mux) {
                         if (sock == this.sock)
                             this.sock = null; // Connection has dead.
+
+                        clear();
                     }
                 }
             }
         }
+
+        /**
+         *
+         */
+        private void clear() {
+            assert Thread.holdsLock(mux);
+
+            queue.clear();
+            unackedMsg = null;
+
+            CountDownLatch forceLeaveLatch = this.forceLeaveLatch;
+
+            if (forceLeaveLatch != null) {
+                this.forceLeaveLatch = null;
+
+                forceLeaveLatch.countDown();
+            }
+        }
     }
 
     /**
@@ -1413,6 +1529,38 @@ else if (msg == SPI_STOP) {
                         else
                             leaveLatch.countDown();
                     }
+                    else if (msg == SPI_RECONNECT) {
+                        if (state == CONNECTED) {
+                            if (reconnector != null) {
+                                reconnector.cancel();
+                                reconnector.join();
+
+                                reconnector = null;
+                            }
+
+                            sockWriter.forceLeave();
+                            sockReader.forceStopRead();
+
+                            currSock = null;
+
+                            queue.clear();
+
+                            onDisconnected();
+
+                            notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
+
+                            UUID newId = UUID.randomUUID();
+
+                            U.quietAndWarn(log, "Local node will try to reconnect to cluster with new id due " +
+                                "to network problems [newId=" + newId +
+                                ", prevId=" + locNode.id() +
+                                ", locNode=" + locNode+ ']');
+
+                            locNode.onClientDisconnected(newId);
+
+                            tryJoin();
+                        }
+                    }
                     else if (msg instanceof TcpDiscoveryNodeFailedMessage &&
                         ((TcpDiscoveryNodeFailedMessage)msg).failedNodeId().equals(locNode.id())) {
                         TcpDiscoveryNodeFailedMessage msg0 = (TcpDiscoveryNodeFailedMessage)msg;
@@ -1495,20 +1643,7 @@ else if (msg == SPI_RECONNECT_FAILED) {
                                         ", failMsg=" + forceFailMsg + ']');
                                 }
 
-                                state = DISCONNECTED;
-
-                                nodeAdded = false;
-
-                                IgniteClientDisconnectedCheckedException err =
-                                    new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
-                                    "client node disconnected.");
-
-                                for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
-                                    GridFutureAdapter<Boolean> fut = e.getValue();
-
-                                    if (pingFuts.remove(e.getKey(), fut))
-                                        fut.onDone(err);
-                                }
+                                onDisconnected();
 
                                 notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
                             }
@@ -1603,6 +1738,26 @@ else if (discoMsg instanceof TcpDiscoveryCheckFailedMessage)
             }
         }
 
+        /**
+         *
+         */
+        private void onDisconnected() {
+            state = DISCONNECTED;
+
+            nodeAdded = false;
+
+            IgniteClientDisconnectedCheckedException err =
+                new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
+                    "client node disconnected.");
+
+            for (Map.Entry<UUID, GridFutureAdapter<Boolean>> e : pingFuts.entrySet()) {
+                GridFutureAdapter<Boolean> fut = e.getValue();
+
+                if (pingFuts.remove(e.getKey(), fut))
+                    fut.onDone(err);
+            }
+        }
+
         /**
          * @throws InterruptedException If interrupted.
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 89a9efa481eaa..dfe614b715026 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1609,6 +1609,11 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
         throw new UnsupportedOperationException();
     }
 
+    /** {@inheritDoc} */
+    @Override public void reconnect() throws IgniteSpiException {
+        throw new UnsupportedOperationException("Reconnect is not supported for server.");
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteSpiThread workerThread() {
         return msgWorker;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index f199c20e17edb..84c2ff28afef0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -258,6 +259,13 @@ protected static String threadStatus(Thread t) {
         return t.isAlive() ? "alive" : "dead";
     }
 
+    /**
+     * Leave cluster and try to join again.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public abstract void reconnect() throws IgniteSpiException;
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      * <p>
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 00ae97d1e302c..a2a47feb31486 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1926,6 +1926,15 @@ boolean isSslEnabled() {
         return ignite().configuration().getSslContextFactory() != null;
     }
 
+    /**
+     * Force reconnect to cluster.
+     *
+     * @throws IgniteSpiException If failed.
+     */
+    public void reconnect() throws IgniteSpiException {
+        impl.reconnect();
+    }
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 0f0165b0f04f9..6cdf465978907 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -700,9 +700,12 @@ public void testReconnectInitialExchangeInProgress() throws Exception {
                 try {
                     Ignition.start(optimize(getConfiguration(getTestGridName(SRV_CNT))));
 
-                    fail();
+                    // Commented due to IGNITE-4473, because
+                    // IgniteClientDisconnectedException won't
+                    // be thrown, but client will reconnect.
+//                    fail();
 
-                    return false;
+                    return true;
                 }
                 catch (IgniteClientDisconnectedException e) {
                     log.info("Expected start error: " + e);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
new file mode 100644
index 0000000000000..a5d42e9a12cba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests client to be able restore connection to cluster if coordination is not available.
+ */
+public class IgniteClientRejoinTest extends GridCommonAbstractTest {
+    /** Block. */
+    private volatile boolean block;
+
+    /** Block all. */
+    private volatile boolean blockAll;
+
+    /** Coordinator. */
+    private volatile ClusterNode crd;
+
+    /** Client reconnect disabled. */
+    private boolean clientReconnectDisabled;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK", "true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.clearProperty("IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        clientReconnectDisabled = false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("client")) {
+            cfg.setCommunicationSpi(new TcpCommunicationSpi());
+
+            TcpDiscoverySpi spi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+            DiscoverySpi dspi = new DiscoverySpi();
+
+            dspi.setIpFinder(spi.getIpFinder());
+
+            cfg.setDiscoverySpi(dspi);
+
+            dspi.setJoinTimeout(60_000);
+            dspi.setClientReconnectDisabled(clientReconnectDisabled);
+
+            cfg.setClientMode(true);
+        }
+
+        // TODO: IGNITE-4833
+        cfg.setPeerClassLoadingEnabled(false);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectAfterStart() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        List<Ignite> clientNodes = new ArrayList<>();
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++)
+            clientNodes.add(startGrid("client" + i));
+
+        blockAll = true;
+
+        GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                U.sleep(5_000);
+
+                block = true;
+                blockAll = false;
+
+                System.out.println(">>> Allow with blocked coordinator.");
+
+                latch.countDown();
+
+                return null;
+            }
+        });
+
+        IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                latch.await();
+
+                U.sleep((new Random().nextInt(15) + 30) * 1000);
+
+                block = false;
+
+                System.out.println(">>> Allow coordinator.");
+
+                return null;
+            }
+        });
+
+        fut.get();
+
+        for (Ignite client : clientNodes) {
+            while (true) {
+                try {
+                    IgniteCache<Integer, Integer> cache = client.getOrCreateCache("some");
+
+                    for (int i = 0; i < 100; i++)
+                        cache.put(i, i);
+
+                    for (int i = 0; i < 100; i++)
+                        assertEquals((Integer)i, cache.get(i));
+
+                    cache.clear();
+
+                    break;
+                }
+                catch (IgniteClientDisconnectedException e) {
+                    e.reconnectFuture().get();
+                }
+            }
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnect() throws Exception {
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        GridTestUtils.runAsync(new Callable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                latch.countDown();
+
+                Random rnd = new Random();
+
+                U.sleep((rnd.nextInt(15) + 15) * 1000);
+
+                block = false;
+
+                System.out.println(">>> ALLOW connection to coordinator.");
+
+                return true;
+            }
+        });
+
+        for (IgniteInternalFuture<Ignite> clientFut : futs) {
+            Ignite client = clientFut.get();
+
+            IgniteCache<Integer, Integer> cache = client.getOrCreateCache(client.name());
+
+            for (int i = 0; i < 100; i++)
+                cache.put(i, i);
+
+            for (int i = 0; i < 100; i++)
+                assert i == cache.get(i);
+        }
+
+        assertEquals(CLIENTS_NUM, srv1.cluster().forClients().nodes().size());
+        assertEquals(CLIENTS_NUM, srv2.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsReconnectDisabled() throws Exception {
+        clientReconnectDisabled = true;
+
+        Ignite srv1 = startGrid("server1");
+
+        crd = ((IgniteKernal)srv1).localNode();
+
+        Ignite srv2 = startGrid("server2");
+
+        block = true;
+
+        List<IgniteInternalFuture<Ignite>> futs = new ArrayList<>();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        final int CLIENTS_NUM = 5;
+
+        for (int i = 0; i < CLIENTS_NUM; i++) {
+            final int idx = i;
+
+            IgniteInternalFuture<Ignite> fut = GridTestUtils.runAsync(new Callable<Ignite>() {
+                @Override public Ignite call() throws Exception {
+                    latch.await();
+
+                    return startGrid("client" + idx);
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        latch.countDown();
+
+        for (final IgniteInternalFuture<Ignite> clientFut : futs) {
+            //noinspection ThrowableNotThrown
+            GridTestUtils.assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    clientFut.get();
+
+                    return null;
+                }
+            }, IgniteCheckedException.class, null);
+        }
+
+        assertEquals(0, srv1.cluster().forClients().nodes().size());
+        assertEquals(0, srv2.cluster().forClients().nodes().size());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 3 * 60_000;
+    }
+
+    /**
+     *
+     */
+    private class TcpCommunicationSpi extends org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (blockAll || block && node.id().equals(crd.id()))
+                throw new IgniteSpiException(new SocketException("Test communication exception"));
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     *
+     */
+    private class DiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, byte[] data,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(TcpDiscoveryAbstractMessage msg, Socket sock, int res,
+            long timeout) throws IOException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            super.writeToSocket(msg, sock, res, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Socket openSocket(Socket sock, InetSocketAddress remAddr,
+            IgniteSpiOperationTimeoutHelper timeoutHelper) throws IOException, IgniteSpiOperationTimeoutException {
+            if (blockAll || block && sock.getPort() == 47500)
+                throw new SocketException("Test discovery exception");
+
+            return super.openSocket(sock, remAddr, timeoutHelper);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 331b581bb7bec..0483a1ce8fbc6 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -32,6 +32,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.IgniteState;
@@ -43,6 +44,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -1788,8 +1790,7 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
         clientNodeIds.add(client.cluster().localNode().id());
 
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override
-            public boolean apply() {
+            @Override public boolean apply() {
                 return srv.cluster().nodes().size() == 2;
             }
         }, awaitTime());
@@ -1799,6 +1800,49 @@ public boolean apply() {
         assertFalse(err.get());
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testForceClientReconnect() throws Exception {
+        startServerNodes(1);
+
+        startClientNodes(1);
+
+        Ignite srv = G.ignite("server-0");
+        IgniteKernal client = (IgniteKernal)G.ignite("client-0");
+
+        UUID clientId = F.first(clientNodeIds);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        srv.events().enableLocal(EVT_NODE_JOINED);
+
+        srv.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                latch.countDown();
+
+                return false;
+            }
+        }, EVT_NODE_JOINED);
+
+        client.context().discovery().reconnect();
+
+        assert latch.await(10, TimeUnit.SECONDS);
+
+        while (true) {
+            try {
+                UUID newId = client.localNode().id();
+
+                assert !clientId.equals(newId) : clientId;
+
+                break;
+            }
+            catch (IgniteClientDisconnectedException e) {
+                e.reconnectFuture().get(10_000);
+            }
+        }
+    }
+
     /**
      * @param ignite Ignite.
      * @throws Exception If failed.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index ea8e37bada1d0..67d88e1e29eac 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.IgniteClientReconnectServicesTest;
 import org.apache.ignite.internal.IgniteClientReconnectStopTest;
 import org.apache.ignite.internal.IgniteClientReconnectStreamerTest;
+import org.apache.ignite.internal.IgniteClientRejoinTest;
 
 /**
  *
@@ -52,6 +53,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteClientReconnectServicesTest.class);
         suite.addTestSuite(IgniteClientReconnectStreamerTest.class);
         suite.addTestSuite(IgniteClientReconnectFailoverTest.class);
+        suite.addTestSuite(IgniteClientRejoinTest.class);
 
         return suite;
     }

From 3a496d82d56b005f2468eab5834e4a45133742b0 Mon Sep 17 00:00:00 2001
From: Sergi Vladykin <sergi.vladykin@gmail.com>
Date: Sat, 18 Mar 2017 14:46:00 +0300
Subject: [PATCH 051/516] ignite-1.9 - test added

---
 .../query/IgniteSqlSplitterSelfTest.java      | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 8eae54950214b..7570184c036f7 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -152,6 +152,34 @@ public void testOffsetLimit() throws Exception {
         }
     }
 
+    /**
+     *
+     */
+    public void testReplicatedOnlyTables() {
+        IgniteCache<Integer,Value> p = ignite(0).getOrCreateCache(cacheConfig("p", true,
+            Integer.class, Value.class));
+        IgniteCache<Integer,Value> r = ignite(0).getOrCreateCache(cacheConfig("r", false,
+            Integer.class, Value.class));
+
+        try {
+            int cnt = 1000;
+
+            for (int i = 0; i < cnt; i++)
+                r.put(i, new Value(i, -i));
+
+            // Query data from replicated table using partitioned cache.
+            assertEquals(cnt, p.query(new SqlFieldsQuery("select 1 from \"r\".Value")).getAll().size());
+
+            List<List<?>> res = p.query(new SqlFieldsQuery("select count(1) from \"r\".Value")).getAll();
+            assertEquals(1, res.size());
+            assertEquals(cnt, ((Number)res.get(0).get(0)).intValue());
+        }
+        finally {
+            p.destroy();
+            r.destroy();
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From c4de164392ddc114c88d5a6eba0ff0b13d32542f Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Mon, 20 Mar 2017 16:31:15 +0300
Subject: [PATCH 052/516] IGNITE-518: Unmuted tests that was fixed in
 ignite-4036. This closes #1636.

---
 .../expiry/IgniteCacheExpiryPolicyAbstractTest.java   | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index b234631e4002a..0b1b8c5fff3a4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -162,7 +162,8 @@ public void testCreateUpdate0() throws Exception {
         assertEquals(0, pSize);
     }
 
-    /**     * @throws Exception If failed.
+    /**
+     * @throws Exception If failed.
      */
     public void testZeroOnCreate() throws Exception {
         factory = CreatedExpiryPolicy.factoryOf(Duration.ZERO);
@@ -849,8 +850,6 @@ private void createUpdate(Integer key, @Nullable TransactionConcurrency txConcur
      * @throws Exception If failed.
      */
     public void testNearCreateUpdate() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-518");
-
         if (cacheMode() != PARTITIONED)
             return;
 
@@ -974,8 +973,6 @@ private void nearPutAll() throws Exception {
      * @throws Exception If failed.
      */
     public void testNearAccess() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-518");
-
         if (cacheMode() != PARTITIONED)
             return;
 
@@ -1023,10 +1020,10 @@ public void testNearAccess() throws Exception {
      * @throws Exception If failed.
      */
     public void testNearExpiresOnClient() throws Exception {
-        if(cacheMode() != PARTITIONED)
+        if (cacheMode() != PARTITIONED)
             return;
 
-        factory =  CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS,1));
+        factory = CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 1));
 
         nearCache = true;
 

From cad46799f34888381137e903899d73805ad11b13 Mon Sep 17 00:00:00 2001
From: Dmitriy Shabalin <dshabalin@gridgain.com>
Date: Thu, 2 Mar 2017 16:05:15 +0700
Subject: [PATCH 053/516] IGNITE-4659 Migration to Webpack 2. Upgrade template
 engine from jade to pug. (cherry picked from commit 1080e68)

---
 modules/web-console/frontend/.eslintrc        |   1 +
 modules/web-console/frontend/.gitignore       |   1 +
 .../web-console/frontend/app/app.config.js    |  12 +-
 modules/web-console/frontend/app/app.js       |   7 +-
 ...og.jade => activities-user-dialog.tpl.pug} |   0
 .../activities-user-dialog/index.js           |   2 +-
 ...epicker.jade => form-field-datepicker.pug} |   8 +-
 .../list-of-registered-users/index.js         |   2 +-
 .../list-of-registered-users.controller.js    |  11 +-
 ....jade => list-of-registered-users.tpl.pug} |   4 +-
 .../ui-grid-header/ui-grid-header.scss        |   1 +
 ...rid-header.jade => ui-grid-header.tpl.pug} |   0
 ...rid-settings.jade => ui-grid-settings.pug} |   0
 .../directives/centered/centered.directive.js |   2 +-
 .../centered/{centered.css => centered.scss}  |   0
 .../information/information.directive.js      |   4 +-
 .../{information.jade => information.pug}     |   0
 .../ui-ace-docker/ui-ace-docker.directive.js  |   4 +-
 .../{ui-ace-docker.jade => ui-ace-docker.pug} |   0
 .../ui-ace-java/ui-ace-java.directive.js      |   4 +-
 .../{ui-ace-java.jade => ui-ace-java.pug}     |   0
 .../ui-ace-pojos/ui-ace-pojos.directive.js    |   4 +-
 .../{ui-ace-pojos.jade => ui-ace-pojos.pug}   |   2 +-
 .../ui-ace-pom/ui-ace-pom.directive.js        |   4 +-
 .../{ui-ace-pom.jade => ui-ace-pom.pug}       |   0
 .../ui-ace-sharp/ui-ace-sharp.directive.js    |   4 +-
 .../{ui-ace-sharp.jade => ui-ace-sharp.pug}   |   0
 .../ui-ace-spring/ui-ace-spring.directive.js  |   4 +-
 .../{ui-ace-spring.jade => ui-ace-spring.pug} |   0
 .../frontend/app/helpers/jade/form.jade       |  28 --
 .../frontend/app/helpers/jade/form.pug        |  28 ++
 ...-checkbox.jade => form-field-checkbox.pug} |  10 +-
 ...-datalist.jade => form-field-datalist.pug} |  10 +-
 ...rm-field-down.jade => form-field-down.pug} |   2 +-
 ...-dropdown.jade => form-field-dropdown.pug} |  12 +-
 ...-feedback.jade => form-field-feedback.pug} |  10 +-
 ...-field-label.jade => form-field-label.pug} |   6 +-
 ...ield-number.jade => form-field-number.pug} |   8 +-
 ...-password.jade => form-field-password.pug} |   8 +-
 ...rm-field-text.jade => form-field-text.pug} |  16 +-
 .../{form-field-up.jade => form-field-up.pug} |   2 +-
 .../form/{form-group.jade => form-group.pug}  |   0
 .../helpers/jade/{mixins.jade => mixins.pug}  |  68 ++--
 .../app/modules/agent/agent.module.js         |   4 +-
 .../modules/branding/header-logo.directive.js |   4 +-
 .../{header-logo.jade => header-logo.pug}     |   0
 .../branding/powered-by-apache.directive.js   |   4 +-
 ...d-by-apache.jade => powered-by-apache.pug} |   0
 .../frontend/app/modules/demo/Demo.module.js  |   3 +-
 .../app/modules/dialog/dialog.factory.js      |   2 +-
 .../dialog/{dialog.jade => dialog.tpl.pug}    |   0
 .../GettingStarted.provider.js                |   3 +-
 .../app/modules/loading/loading.directive.js  |   8 +-
 .../loading/{loading.jade => loading.pug}     |   0
 .../loading/{loading.css => loading.scss}     |   0
 .../app/modules/nodes/Nodes.service.js        |   4 +-
 ...nodes-dialog.jade => nodes-dialog.tpl.pug} |   0
 .../app/modules/sql/notebook.controller.js    |   4 +-
 .../app/modules/sql/sql.controller.js         |  15 +-
 .../frontend/app/modules/sql/sql.module.js    |  15 +-
 .../app/modules/states/admin.state.js         |   4 +-
 .../app/modules/states/configuration.state.js |  23 +-
 .../caches/{affinity.jade => affinity.pug}    |  24 +-
 ...-near-cache.jade => client-near-cache.pug} |  14 +-
 .../{concurrency.jade => concurrency.pug}     |  18 +-
 .../caches/{general.jade => general.pug}      |  24 +-
 .../caches/{memory.jade => memory.pug}        |  27 +-
 ...ache-client.jade => near-cache-client.pug} |  18 +-
 ...ache-server.jade => near-cache-server.pug} |  18 +-
 .../{node-filter.jade => node-filter.pug}     |  24 +-
 .../caches/{query.jade => query.pug}          |  40 +--
 .../caches/{rebalance.jade => rebalance.pug}  |  24 +-
 .../{statistics.jade => statistics.pug}       |  12 +-
 .../caches/{store.jade => store.pug}          | 104 +++----
 .../clusters/{atomic.jade => atomic.pug}      |  14 +-
 .../{attributes.jade => attributes.pug}       |  16 +-
 .../clusters/{binary.jade => binary.pug}      |  14 +-
 .../{cache-key-cfg.jade => cache-key-cfg.pug} |  14 +-
 .../{checkpoint.jade => checkpoint.pug}       |  20 +-
 .../clusters/checkpoint/{fs.jade => fs.pug}   |  14 +-
 .../checkpoint/{jdbc.jade => jdbc.pug}        |   2 +-
 .../clusters/checkpoint/{s3.jade => s3.pug}   |   2 +-
 .../{collision.jade => collision.pug}         |  26 +-
 .../collision/{custom.jade => custom.pug}     |   4 +-
 .../{fifo-queue.jade => fifo-queue.pug}       |   6 +-
 .../{job-stealing.jade => job-stealing.pug}   |  22 +-
 ...priority-queue.jade => priority-queue.pug} |  16 +-
 .../{communication.jade => communication.pug} |  58 ++--
 .../{connector.jade => connector.pug}         |  46 +--
 .../{deployment.jade => deployment.pug}       |  72 ++---
 .../{discovery.jade => discovery.pug}         |  54 ++--
 .../clusters/{events.jade => events.pug}      |  18 +-
 .../clusters/{failover.jade => failover.pug}  |  12 +-
 .../clusters/{general.jade => general.pug}    |  42 +--
 .../discovery/{cloud.jade => cloud.pug}       |  48 +--
 .../discovery/{google.jade => google.pug}     |  12 +-
 .../general/discovery/{jdbc.jade => jdbc.pug} |   8 +-
 .../{multicast.jade => multicast.pug}         |  30 +-
 .../general/discovery/{s3.jade => s3.pug}     |   6 +-
 .../discovery/{shared.jade => shared.pug}     |   4 +-
 .../general/discovery/{vm.jade => vm.pug}     |  20 +-
 .../{zookeeper.jade => zookeeper.pug}         |  48 +--
 ...f.jade => bounded-exponential-backoff.pug} |   8 +-
 .../retrypolicy/{custom.jade => custom.pug}   |   6 +-
 ...l-backoff.jade => exponential-backoff.pug} |   8 +-
 .../retrypolicy/{forever.jade => forever.pug} |   4 +-
 .../retrypolicy/{n-times.jade => n-times.pug} |   6 +-
 .../{one-time.jade => one-time.pug}           |   4 +-
 .../{until-elapsed.jade => until-elapsed.pug} |   6 +-
 .../clusters/{igfs.jade => igfs.pug}          |  10 +-
 ...load-balancing.jade => load-balancing.pug} |  12 +-
 .../clusters/{logger.jade => logger.pug}      |  22 +-
 .../logger/{custom.jade => custom.pug}        |   4 +-
 .../clusters/logger/{log4j.jade => log4j.pug} |   8 +-
 .../logger/{log4j2.jade => log4j2.pug}        |   6 +-
 .../{marshaller.jade => marshaller.pug}       |  26 +-
 .../clusters/{metrics.jade => metrics.pug}    |  16 +-
 .../clusters/{odbc.jade => odbc.pug}          |  12 +-
 .../clusters/{ssl.jade => ssl.pug}            |  44 +--
 .../clusters/{swap.jade => swap.pug}          |  28 +-
 .../clusters/{thread.jade => thread.pug}      |  18 +-
 .../clusters/{time.jade => time.pug}          |  16 +-
 .../{transactions.jade => transactions.pug}   |  20 +-
 .../domains/{general.jade => general.pug}     |  14 +-
 .../domains/{query.jade => query.pug}         |  60 ++--
 .../domains/{store.jade => store.pug}         |  60 ++--
 .../igfs/{dual.jade => dual.pug}              |  14 +-
 .../{fragmentizer.jade => fragmentizer.pug}   |  16 +-
 .../igfs/{general.jade => general.pug}        |   8 +-
 .../configuration/igfs/{ipc.jade => ipc.pug}  |  24 +-
 .../igfs/{misc.jade => misc.pug}              |  56 ++--
 .../igfs/{secondary.jade => secondary.pug}    |  18 +-
 .../summary/summary.controller.js             |   5 +
 .../app/modules/states/errors.state.js        |   4 +-
 .../app/modules/states/password.state.js      |   6 +-
 .../app/modules/states/profile.state.js       |   4 +-
 .../app/modules/states/signin.state.js        |   2 +-
 .../frontend/app/services/Clone.service.js    |   4 +-
 .../frontend/app/services/Confirm.service.js  |   4 +-
 .../app/services/ConfirmBatch.service.js      |   4 +-
 .../frontend/controllers/caches-controller.js |   4 +-
 .../controllers/domains-controller.js         |   4 +-
 .../frontend/gulpfile.babel.js/paths.js       |  28 +-
 .../frontend/gulpfile.babel.js/tasks/build.js |   2 +-
 .../gulpfile.babel.js/tasks/bundle.js         |   8 +-
 .../frontend/gulpfile.babel.js/tasks/watch.js |   5 +-
 .../gulpfile.babel.js/webpack/common.js       | 292 +++++++++---------
 .../webpack/environments/development.js       |  90 +++---
 .../webpack/environments/production.js        |  41 ++-
 .../gulpfile.babel.js/webpack/index.js        |   4 +-
 modules/web-console/frontend/package.json     |  60 ++--
 .../frontend/test/e2e/exampe.test.js          |   4 +-
 .../frontend/test/karma.conf.babel.js         |   7 +-
 .../frontend/test/protractor.conf.js          |  16 +-
 .../test/unit/JavaTransformer.test.js         |   2 +-
 .../frontend/test/unit/JavaTypes.test.js      | 137 ++++----
 .../test/unit/SharpTransformer.test.js        |   2 +-
 .../test/unit/SpringTransformer.test.js       |   2 +-
 .../frontend/test/unit/SqlTypes.test.js       |   7 +-
 .../frontend/test/unit/UserAuth.test.js       |   4 +-
 .../frontend/test/unit/Version.test.js        |   1 +
 .../test/unit/defaultName.filter.test.js      |  27 +-
 .../frontend/views/{403.jade => 403.pug}      |   0
 .../frontend/views/{404.jade => 404.pug}      |   0
 .../frontend/views/{base.jade => base.pug}    |   4 +-
 .../{caches.jade => caches.tpl.pug}           |  24 +-
 .../{clusters.jade => clusters.tpl.pug}       |  50 +--
 ...ins-import.jade => domains-import.tpl.pug} |   4 +-
 .../{domains.jade => domains.tpl.pug}         |   8 +-
 .../configuration/{igfs.jade => igfs.tpl.pug} |  14 +-
 .../{sidebar.jade => sidebar.tpl.pug}         |   0
 ...jade => summary-project-structure.tpl.pug} |   1 +
 .../{summary-tabs.jade => summary-tabs.pug}   |   2 +-
 .../{summary.jade => summary.tpl.pug}         |   8 +-
 .../includes/{footer.jade => footer.pug}      |   0
 .../includes/{header.jade => header.pug}      |   0
 .../frontend/views/{index.jade => index.pug}  |   0
 .../views/{reset.jade => reset.tpl.pug}       |   0
 .../settings/{admin.jade => admin.tpl.pug}    |   0
 .../{profile.jade => profile.tpl.pug}         |   0
 .../views/{signin.jade => signin.tpl.pug}     |   0
 ...e-metadata.jade => cache-metadata.tpl.pug} |   0
 ...t-settings.jade => chart-settings.tpl.pug} |   0
 ...notebook-new.jade => notebook-new.tpl.pug} |   0
 ...graph-rate.jade => paragraph-rate.tpl.pug} |   0
 .../views/sql/{sql.jade => sql.tpl.pug}       |  18 +-
 ...t-download.jade => agent-download.tpl.pug} |   0
 .../templates/{alert.jade => alert.tpl.pug}   |   0
 ...tch-confirm.jade => batch-confirm.tpl.pug} |   0
 .../templates/{clone.jade => clone.tpl.pug}   |   2 +-
 .../{confirm.jade => confirm.tpl.pug}         |   0
 .../{demo-info.jade => demo-info.tpl.pug}     |   0
 .../{dropdown.jade => dropdown.tpl.pug}       |   0
 ...g-started.jade => getting-started.tpl.pug} |   0
 .../{message.jade => message.tpl.pug}         |   0
 .../frontend/views/templates/pagination.jade  |  32 --
 .../templates/{select.jade => select.tpl.pug} |   0
 ...on-error.jade => validation-error.tpl.pug} |   0
 198 files changed, 1442 insertions(+), 1436 deletions(-)
 rename modules/web-console/frontend/app/components/activities-user-dialog/{activities-user-dialog.jade => activities-user-dialog.tpl.pug} (100%)
 rename modules/web-console/frontend/app/components/form-field-datepicker/{form-field-datepicker.jade => form-field-datepicker.pug} (90%)
 rename modules/web-console/frontend/app/components/list-of-registered-users/{list-of-registered-users.jade => list-of-registered-users.tpl.pug} (98%)
 rename modules/web-console/frontend/app/components/ui-grid-header/{ui-grid-header.jade => ui-grid-header.tpl.pug} (100%)
 rename modules/web-console/frontend/app/components/ui-grid-settings/{ui-grid-settings.jade => ui-grid-settings.pug} (100%)
 rename modules/web-console/frontend/app/directives/centered/{centered.css => centered.scss} (100%)
 rename modules/web-console/frontend/app/directives/information/{information.jade => information.pug} (100%)
 rename modules/web-console/frontend/app/directives/ui-ace-docker/{ui-ace-docker.jade => ui-ace-docker.pug} (100%)
 rename modules/web-console/frontend/app/directives/ui-ace-java/{ui-ace-java.jade => ui-ace-java.pug} (100%)
 rename modules/web-console/frontend/app/directives/ui-ace-pojos/{ui-ace-pojos.jade => ui-ace-pojos.pug} (97%)
 rename modules/web-console/frontend/app/directives/ui-ace-pom/{ui-ace-pom.jade => ui-ace-pom.pug} (100%)
 rename modules/web-console/frontend/app/directives/ui-ace-sharp/{ui-ace-sharp.jade => ui-ace-sharp.pug} (100%)
 rename modules/web-console/frontend/app/directives/ui-ace-spring/{ui-ace-spring.jade => ui-ace-spring.pug} (100%)
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form.jade
 create mode 100644 modules/web-console/frontend/app/helpers/jade/form.pug
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-checkbox.jade => form-field-checkbox.pug} (83%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-datalist.jade => form-field-datalist.pug} (86%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-down.jade => form-field-down.pug} (95%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-dropdown.jade => form-field-dropdown.pug} (80%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-feedback.jade => form-field-feedback.pug} (83%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-label.jade => form-field-label.pug} (88%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-number.jade => form-field-number.pug} (91%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-password.jade => form-field-password.pug} (91%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-text.jade => form-field-text.pug} (86%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-field-up.jade => form-field-up.pug} (95%)
 rename modules/web-console/frontend/app/helpers/jade/form/{form-group.jade => form-group.pug} (100%)
 rename modules/web-console/frontend/app/helpers/jade/{mixins.jade => mixins.pug} (91%)
 rename modules/web-console/frontend/app/modules/branding/{header-logo.jade => header-logo.pug} (100%)
 rename modules/web-console/frontend/app/modules/branding/{powered-by-apache.jade => powered-by-apache.pug} (100%)
 rename modules/web-console/frontend/app/modules/dialog/{dialog.jade => dialog.tpl.pug} (100%)
 rename modules/web-console/frontend/app/modules/loading/{loading.jade => loading.pug} (100%)
 rename modules/web-console/frontend/app/modules/loading/{loading.css => loading.scss} (100%)
 rename modules/web-console/frontend/app/modules/nodes/{nodes-dialog.jade => nodes-dialog.tpl.pug} (100%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{affinity.jade => affinity.pug} (77%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{client-near-cache.jade => client-near-cache.pug} (81%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{concurrency.jade => concurrency.pug} (79%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{general.jade => general.pug} (77%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{memory.jade => memory.pug} (86%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{near-cache-client.jade => near-cache-client.pug} (76%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{near-cache-server.jade => near-cache-server.pug} (77%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{node-filter.jade => node-filter.pug} (71%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{query.jade => query.pug} (76%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{rebalance.jade => rebalance.pug} (75%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{statistics.jade => statistics.pug} (76%)
 rename modules/web-console/frontend/app/modules/states/configuration/caches/{store.jade => store.pug} (72%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{atomic.jade => atomic.pug} (83%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{attributes.jade => attributes.pug} (80%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{binary.jade => binary.pug} (92%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{cache-key-cfg.jade => cache-key-cfg.pug} (83%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{checkpoint.jade => checkpoint.pug} (86%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/{fs.jade => fs.pug} (85%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/{jdbc.jade => jdbc.pug} (98%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/{s3.jade => s3.pug} (99%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{collision.jade => collision.pug} (75%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/collision/{custom.jade => custom.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/collision/{fifo-queue.jade => fifo-queue.pug} (78%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/collision/{job-stealing.jade => job-stealing.pug} (73%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/collision/{priority-queue.jade => priority-queue.pug} (64%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{communication.jade => communication.pug} (56%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{connector.jade => connector.pug} (71%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{deployment.jade => deployment.pug} (78%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{discovery.jade => discovery.pug} (50%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{events.jade => events.pug} (78%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{failover.jade => failover.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{general.jade => general.pug} (71%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{cloud.jade => cloud.pug} (78%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{google.jade => google.pug} (68%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{jdbc.jade => jdbc.pug} (82%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{multicast.jade => multicast.pug} (79%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{s3.jade => s3.pug} (79%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{shared.jade => shared.pug} (89%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{vm.jade => vm.pug} (83%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/{zookeeper.jade => zookeeper.pug} (70%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{bounded-exponential-backoff.jade => bounded-exponential-backoff.pug} (84%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{custom.jade => custom.pug} (89%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{exponential-backoff.jade => exponential-backoff.pug} (83%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{forever.jade => forever.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{n-times.jade => n-times.pug} (86%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{one-time.jade => one-time.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/{until-elapsed.jade => until-elapsed.pug} (86%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{igfs.jade => igfs.pug} (84%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{load-balancing.jade => load-balancing.pug} (94%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{logger.jade => logger.pug} (80%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/logger/{custom.jade => custom.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/logger/{log4j.jade => log4j.pug} (87%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/logger/{log4j2.jade => log4j2.pug} (90%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{marshaller.jade => marshaller.pug} (76%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{metrics.jade => metrics.pug} (73%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{odbc.jade => odbc.pug} (83%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{ssl.jade => ssl.pug} (73%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{swap.jade => swap.pug} (73%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{thread.jade => thread.pug} (68%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{time.jade => time.pug} (73%)
 rename modules/web-console/frontend/app/modules/states/configuration/clusters/{transactions.jade => transactions.pug} (78%)
 rename modules/web-console/frontend/app/modules/states/configuration/domains/{general.jade => general.pug} (79%)
 rename modules/web-console/frontend/app/modules/states/configuration/domains/{query.jade => query.pug} (77%)
 rename modules/web-console/frontend/app/modules/states/configuration/domains/{store.jade => store.pug} (74%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{dual.jade => dual.pug} (75%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{fragmentizer.jade => fragmentizer.pug} (70%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{general.jade => general.pug} (89%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{ipc.jade => ipc.pug} (66%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{misc.jade => misc.pug} (66%)
 rename modules/web-console/frontend/app/modules/states/configuration/igfs/{secondary.jade => secondary.pug} (69%)
 rename modules/web-console/frontend/views/{403.jade => 403.pug} (100%)
 rename modules/web-console/frontend/views/{404.jade => 404.pug} (100%)
 rename modules/web-console/frontend/views/{base.jade => base.pug} (94%)
 rename modules/web-console/frontend/views/configuration/{caches.jade => caches.tpl.pug} (89%)
 rename modules/web-console/frontend/views/configuration/{clusters.jade => clusters.tpl.pug} (83%)
 rename modules/web-console/frontend/views/configuration/{domains-import.jade => domains-import.tpl.pug} (99%)
 rename modules/web-console/frontend/views/configuration/{domains.jade => domains.tpl.pug} (97%)
 rename modules/web-console/frontend/views/configuration/{igfs.jade => igfs.tpl.pug} (94%)
 rename modules/web-console/frontend/views/configuration/{sidebar.jade => sidebar.tpl.pug} (100%)
 rename modules/web-console/frontend/views/configuration/{summary-project-structure.jade => summary-project-structure.tpl.pug} (99%)
 rename modules/web-console/frontend/views/configuration/{summary-tabs.jade => summary-tabs.pug} (97%)
 rename modules/web-console/frontend/views/configuration/{summary.jade => summary.tpl.pug} (93%)
 rename modules/web-console/frontend/views/includes/{footer.jade => footer.pug} (100%)
 rename modules/web-console/frontend/views/includes/{header.jade => header.pug} (100%)
 rename modules/web-console/frontend/views/{index.jade => index.pug} (100%)
 rename modules/web-console/frontend/views/{reset.jade => reset.tpl.pug} (100%)
 rename modules/web-console/frontend/views/settings/{admin.jade => admin.tpl.pug} (100%)
 rename modules/web-console/frontend/views/settings/{profile.jade => profile.tpl.pug} (100%)
 rename modules/web-console/frontend/views/{signin.jade => signin.tpl.pug} (100%)
 rename modules/web-console/frontend/views/sql/{cache-metadata.jade => cache-metadata.tpl.pug} (100%)
 rename modules/web-console/frontend/views/sql/{chart-settings.jade => chart-settings.tpl.pug} (100%)
 rename modules/web-console/frontend/views/sql/{notebook-new.jade => notebook-new.tpl.pug} (100%)
 rename modules/web-console/frontend/views/sql/{paragraph-rate.jade => paragraph-rate.tpl.pug} (100%)
 rename modules/web-console/frontend/views/sql/{sql.jade => sql.tpl.pug} (95%)
 rename modules/web-console/frontend/views/templates/{agent-download.jade => agent-download.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{alert.jade => alert.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{batch-confirm.jade => batch-confirm.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{clone.jade => clone.tpl.pug} (98%)
 rename modules/web-console/frontend/views/templates/{confirm.jade => confirm.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{demo-info.jade => demo-info.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{dropdown.jade => dropdown.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{getting-started.jade => getting-started.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{message.jade => message.tpl.pug} (100%)
 delete mode 100644 modules/web-console/frontend/views/templates/pagination.jade
 rename modules/web-console/frontend/views/templates/{select.jade => select.tpl.pug} (100%)
 rename modules/web-console/frontend/views/templates/{validation-error.jade => validation-error.tpl.pug} (100%)

diff --git a/modules/web-console/frontend/.eslintrc b/modules/web-console/frontend/.eslintrc
index 988cfa07c0888..958c6d11ef40f 100644
--- a/modules/web-console/frontend/.eslintrc
+++ b/modules/web-console/frontend/.eslintrc
@@ -32,6 +32,7 @@ globals:
     $generatorOptional: true
     saveAs: true
     process: true
+    require: true
 
 rules:
     arrow-parens: [1, "always"]
diff --git a/modules/web-console/frontend/.gitignore b/modules/web-console/frontend/.gitignore
index 46bca1383a528..27138f6a6875c 100644
--- a/modules/web-console/frontend/.gitignore
+++ b/modules/web-console/frontend/.gitignore
@@ -1,5 +1,6 @@
 *.idea
 *.log
+*.log.*
 .npmrc
 build/*
 node_modules
diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js
index 39d761f418eef..3ca5c3b8eaef2 100644
--- a/modules/web-console/frontend/app/app.config.js
+++ b/modules/web-console/frontend/app/app.config.js
@@ -16,6 +16,7 @@
  */
 
 import _ from 'lodash';
+import angular from 'angular';
 
 const nonNil = _.negate(_.isNil);
 const nonEmpty = _.negate(_.isEmpty);
@@ -25,7 +26,10 @@ _.mixin({
     nonEmpty
 });
 
-import alertTemplateUrl from '../views/templates/alert.jade';
+import alertTemplateUrl from 'views/templates/alert.tpl.pug';
+import selectTemplateUrl from 'views/templates/select.tpl.pug';
+import dropdownTemplateUrl from 'views/templates/dropdown.tpl.pug';
+import validationTemplateUrl from 'views/templates/validation-error.tpl.pug';
 
 const igniteConsoleCfg = angular.module('ignite-console.config', ['ngAnimate', 'mgcrea.ngStrap']);
 
@@ -48,7 +52,7 @@ igniteConsoleCfg.config(['$popoverProvider', ($popoverProvider) => {
         trigger: 'manual',
         placement: 'right',
         container: 'body',
-        templateUrl: '/templates/validation-error.html'
+        templateUrl: validationTemplateUrl
     });
 }]);
 
@@ -70,7 +74,7 @@ igniteConsoleCfg.config(['$selectProvider', ($selectProvider) => {
         maxLength: '5',
         allText: 'Select All',
         noneText: 'Clear All',
-        templateUrl: '/templates/select.html',
+        templateUrl: selectTemplateUrl,
         iconCheckmark: 'fa fa-check',
         caretHtml: ''
     });
@@ -91,7 +95,7 @@ igniteConsoleCfg.config(['$alertProvider', ($alertProvider) => {
 // AngularStrap dropdowns () configuration.
 igniteConsoleCfg.config(['$dropdownProvider', ($dropdownProvider) => {
     angular.extend($dropdownProvider.defaults, {
-        templateUrl: 'templates/dropdown.html'
+        templateUrl: dropdownTemplateUrl
     });
 }]);
 
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 9958cb53ae539..5e3bb07434b2c 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -19,7 +19,6 @@ import '../public/stylesheets/style.scss';
 import '../app/components/ui-grid-header/ui-grid-header.scss';
 import '../app/components/ui-grid-settings/ui-grid-settings.scss';
 import '../app/components/form-field-datepicker/form-field-datepicker.scss';
-import './helpers/jade/mixins.jade';
 
 import './app.config';
 
@@ -122,7 +121,7 @@ import IgniteActivitiesUserDialog from './components/activities-user-dialog';
 // Inject external modules.
 import 'ignite_modules_temp/index';
 
-import baseTemplate from '../views/base.jade';
+import baseTemplate from 'views/base.pug';
 
 angular
 .module('ignite-console', [
@@ -242,12 +241,12 @@ angular
         .state('base', {
             url: '',
             abstract: true,
-            templateUrl: baseTemplate
+            template: baseTemplate
         })
         .state('settings', {
             url: '/settings',
             abstract: true,
-            templateUrl: baseTemplate
+            template: baseTemplate
         });
 
     $urlRouterProvider.otherwise('/404');
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
rename to modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/index.js b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
index dca6ba92ede6d..2f8fdeff00473 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/index.js
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
@@ -16,7 +16,7 @@
  */
 
  import controller from './activities-user-dialog.controller';
- import templateUrl from './activities-user-dialog.jade';
+ import templateUrl from './activities-user-dialog.tpl.pug';
 
  export default ['$modal', ($modal) => ({ show = true, user }) => {
      const ActivitiesUserDialog = $modal({
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
similarity index 90%
rename from modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
rename to modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
index 2578cf4b17189..c9d382cb36cc4 100644
--- a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
+++ b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
@@ -17,15 +17,15 @@
 mixin ignite-form-field-datepicker(label, model, name, disabled, required, placeholder, tip)
     mixin form-field-input()
         input.form-control(
-            id='{{ #{name} }}Input'
-            name='{{ #{name} }}'
+            id=`{{ ${name} }}Input`
+            name=`{{ ${name} }}`
 
             placeholder=placeholder
             
             data-ng-model=model
 
-            data-ng-required=required && '#{required}'
-            data-ng-disabled=disabled && '#{disabled}'
+            data-ng-required=required && `${required}`
+            data-ng-disabled=disabled && `${disabled}`
 
             bs-datepicker
             data-date-format='MMM yyyy'
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/index.js b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
index 32a34f41c02d6..22a89da3b6a24 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/index.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './list-of-registered-users.jade';
+import templateUrl from './list-of-registered-users.tpl.pug';
 import controller from './list-of-registered-users.controller';
 
 export default [() => {
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 272681a5f576b..33fe819068c78 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import headerTemplate from 'app/components/ui-grid-header/ui-grid-header.jade';
+import headerTemplate from 'app/components/ui-grid-header/ui-grid-header.tpl.pug';
 
 import columnDefs from './list-of-registered-users.column-defs';
 import categories from './list-of-registered-users.categories';
@@ -30,9 +30,9 @@ const rowTemplate = `<div
   ui-grid-cell/>`;
 
 export default class IgniteListOfRegisteredUsersCtrl {
-    static $inject = ['$scope', '$state', '$filter', '$templateCache', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
+    static $inject = ['$scope', '$state', '$filter', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
 
-    constructor($scope, $state, $filter, $templateCache, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
+    constructor($scope, $state, $filter, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
         const $ctrl = this;
 
         const companySelectOptions = [];
@@ -113,7 +113,7 @@ export default class IgniteListOfRegisteredUsersCtrl {
             columnVirtualizationThreshold: 30,
             columnDefs,
             categories,
-            headerTemplate: $templateCache.get(headerTemplate),
+            headerTemplate,
             rowTemplate,
             enableFiltering: true,
             enableRowSelection: true,
@@ -157,6 +157,9 @@ export default class IgniteListOfRegisteredUsersCtrl {
             AdminData.loadUsers(params)
                 .then((data) => $ctrl.gridOptions.data = data)
                 .then((data) => {
+                    companySelectOptions.length = 0;
+                    countrySelectOptions.length = 0;
+
                     companySelectOptions.push(...usersToFilterOptions('company'));
                     countrySelectOptions.push(...usersToFilterOptions('countryCode'));
 
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
similarity index 98%
rename from modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
rename to modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
index 119591036d9da..52975b98dc0ee 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
@@ -14,8 +14,8 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
-include /app/components/form-field-datepicker/form-field-datepicker.jade
+include /app/helpers/jade/mixins
+include /app/components/form-field-datepicker/form-field-datepicker.pug
 
 mixin grid-settings()
     i.fa.fa-bars(data-animation='am-flip-x' bs-dropdown='' aria-haspopup='true' aria-expanded='expanded' data-auto-close='1' data-trigger='click')
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
index c39050469db9c..c6e7bdf241e98 100644
--- a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
@@ -50,6 +50,7 @@
     .ui-grid-header-span {
         position: relative;
         border-right: 0;
+        background: #f5f5f5;
 
         .ng-hide + .ui-grid-header-cell-row .ui-grid-header-cell {
             height: 58px;
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
rename to modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug
diff --git a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.jade b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.pug
similarity index 100%
rename from modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.jade
rename to modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.pug
diff --git a/modules/web-console/frontend/app/directives/centered/centered.directive.js b/modules/web-console/frontend/app/directives/centered/centered.directive.js
index 4abd086e6a5cb..77bbb940fe937 100644
--- a/modules/web-console/frontend/app/directives/centered/centered.directive.js
+++ b/modules/web-console/frontend/app/directives/centered/centered.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import './centered.css';
+import './centered.scss';
 
 export default ['centered', [() => {
     return {
diff --git a/modules/web-console/frontend/app/directives/centered/centered.css b/modules/web-console/frontend/app/directives/centered/centered.scss
similarity index 100%
rename from modules/web-console/frontend/app/directives/centered/centered.css
rename to modules/web-console/frontend/app/directives/centered/centered.scss
diff --git a/modules/web-console/frontend/app/directives/information/information.directive.js b/modules/web-console/frontend/app/directives/information/information.directive.js
index a9a2f8c27fde9..6f304ef8a9b0b 100644
--- a/modules/web-console/frontend/app/directives/information/information.directive.js
+++ b/modules/web-console/frontend/app/directives/information/information.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './information.jade';
+import template from './information.pug';
 
 export default ['igniteInformation', [() => {
     return {
@@ -23,7 +23,7 @@ export default ['igniteInformation', [() => {
             title: '@'
         },
         restrict: 'E',
-        templateUrl,
+        template,
         replace: true,
         transclude: true
     };
diff --git a/modules/web-console/frontend/app/directives/information/information.jade b/modules/web-console/frontend/app/directives/information/information.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/information/information.jade
rename to modules/web-console/frontend/app/directives/information/information.pug
diff --git a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js
index 08e4f76e3bbea..9042acb767f53 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './ui-ace-docker.jade';
+import template from './ui-ace-docker.pug';
 import controller from './ui-ace-docker.controller';
 
 export default ['igniteUiAceDocker', [() => {
@@ -38,7 +38,7 @@ export default ['igniteUiAceDocker', [() => {
             data: '=ngModel'
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['?^igniteUiAceTabs']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.jade b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.jade
rename to modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.pug
diff --git a/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js b/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js
index c21bff7279b59..5f43b2351e1f4 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './ui-ace-java.jade';
+import template from './ui-ace-java.pug';
 import controller from './ui-ace-java.controller';
 
 export default ['igniteUiAceJava', [() => {
@@ -55,7 +55,7 @@ export default ['igniteUiAceJava', [() => {
             client: '@'
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['igniteUiAceJava', '?^igniteUiAceTabs', '?^form', '?ngModel']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.jade b/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.jade
rename to modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.pug
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js
index 7c224b79eb39e..8a8d047de561c 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './ui-ace-pojos.jade';
+import template from './ui-ace-pojos.pug';
 import controller from './ui-ace-pojos.controller';
 
 export default ['igniteUiAcePojos', [() => {
@@ -38,7 +38,7 @@ export default ['igniteUiAcePojos', [() => {
             pojos: '=ngModel'
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['?^igniteUiAceTabs']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.jade b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.pug
similarity index 97%
rename from modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.jade
rename to modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.pug
index 581b8c1451601..4bee5e5f43035 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.jade
+++ b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.pug
@@ -15,7 +15,7 @@
     limitations under the License.
 
 mixin check-tooltip(message)
-    i.tipLabel.icon-help(bs-tooltip='"#{message}"')
+    i.tipLabel.icon-help(bs-tooltip=`"${message}"`)
 
 .panel-details-noborder
     .details-row
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js
index 2a7a87890ace9..664d3a000fd4a 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './ui-ace-pom.jade';
+import template from './ui-ace-pom.pug';
 import controller from './ui-ace-pom.controller';
 
 export default ['igniteUiAcePom', [() => {
@@ -33,7 +33,7 @@ export default ['igniteUiAcePom', [() => {
             cluster: '='
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['?^igniteUiAceTabs']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.jade b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.jade
rename to modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.pug
diff --git a/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js b/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js
index 5d9ad7988b23f..5a37b80742ea8 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './ui-ace-sharp.jade';
+import template from './ui-ace-sharp.pug';
 import controller from './ui-ace-sharp.controller';
 
 export default ['igniteUiAceSharp', ['IgniteSharpTransformer', (generator) => {
@@ -125,7 +125,7 @@ export default ['igniteUiAceSharp', ['IgniteSharpTransformer', (generator) => {
             data: '=?ngModel'
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['igniteUiAceSharp', '?^igniteUiAceTabs', '?^form', '?ngModel']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.jade b/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.jade
rename to modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.pug
diff --git a/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js b/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js
index 42d25b65f3993..de3f434dddca2 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js
@@ -17,7 +17,7 @@
 
 import _ from 'lodash';
 
-import templateUrl from './ui-ace-spring.jade';
+import template from './ui-ace-spring.pug';
 import controller from './ui-ace-spring.controller';
 
 export default ['igniteUiAceSpring', [() => {
@@ -58,7 +58,7 @@ export default ['igniteUiAceSpring', [() => {
             client: '@'
         },
         link,
-        templateUrl,
+        template,
         controller,
         controllerAs: 'ctrl',
         require: ['igniteUiAceSpring', '?^igniteUiAceTabs', '?^form', '?ngModel']
diff --git a/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.jade b/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.pug
similarity index 100%
rename from modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.jade
rename to modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.pug
diff --git a/modules/web-console/frontend/app/helpers/jade/form.jade b/modules/web-console/frontend/app/helpers/jade/form.jade
deleted file mode 100644
index c0c558d9cb925..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form.jade
+++ /dev/null
@@ -1,28 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-include ./form/form-field-feedback.jade
-include ./form/form-field-label.jade
-include ./form/form-field-text.jade
-include ./form/form-field-password.jade
-include ./form/form-field-dropdown.jade
-include ./form/form-field-datalist.jade
-include ./form/form-field-checkbox.jade
-include ./form/form-field-number.jade
-include ./form/form-field-up.jade
-include ./form/form-field-down.jade
-
-include ./form/form-group.jade
diff --git a/modules/web-console/frontend/app/helpers/jade/form.pug b/modules/web-console/frontend/app/helpers/jade/form.pug
new file mode 100644
index 0000000000000..6fddbf6b97687
--- /dev/null
+++ b/modules/web-console/frontend/app/helpers/jade/form.pug
@@ -0,0 +1,28 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include ./form/form-field-feedback
+include ./form/form-field-label
+include ./form/form-field-text
+include ./form/form-field-password
+include ./form/form-field-dropdown
+include ./form/form-field-datalist
+include ./form/form-field-checkbox
+include ./form/form-field-number
+include ./form/form-field-up
+include ./form/form-field-down
+
+include ./form/form-group
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug
similarity index 83%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug
index 222ecfee76317..68a2dd2ece56f 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug
@@ -16,19 +16,19 @@
 
 mixin form-field-checkbox(label, model, name, disabled, required, tip)
     div.checkbox.col-xs-12.col-sm-12.col-md-12
-        label(id='{{ #{name} }}Label')
+        label(id=`{{ ${name} }}Label`)
             .input-tip
                 if block
                     block
                 else
                     input(
-                        id='{{ #{name} }}Input'
-                        name='{{ #{name} }}'
+                        id=`{{ ${name} }}Input`
+                        name=`{{ ${name} }}`
                         type='checkbox'
 
                         data-ng-model=model
-                        data-ng-required=required && '#{required}'
-                        data-ng-disabled=disabled && '#{disabled}'
+                        data-ng-required=required && `${required}`
+                        data-ng-disabled=disabled && `${disabled}`
 
                         data-ng-focus='tableReset()'
 
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.pug
similarity index 86%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.pug
index 4c1970e14bfa0..e4b71afb4315b 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.pug
@@ -19,17 +19,17 @@ mixin form-field-datalist(label, model, name, disabled, required, placeholder, o
 
     mixin form-field-input()
         input.form-control(
-            id='{{ #{name} }}Input'
-            name='{{ #{name} }}'
+            id=`{{ ${name} }}Input`
+            name=`{{ ${name} }}`
             placeholder=placeholder
            
             data-ng-model=model
 
-            data-ng-required=required && '#{required}'
-            data-ng-disabled=disabled && '#{disabled}' || '!#{options}.length'
+            data-ng-required=required && `${required}`
+            data-ng-disabled=disabled && `${disabled}` || `!${options}.length`
 
             bs-typeahead
-            bs-options='item for item in #{options}'
+            bs-options=`item for item in ${options}`
             container='body'
             data-min-length='1'
             ignite-retain-selection
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-down.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-down.pug
similarity index 95%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-down.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-down.pug
index cd10ebee1f889..9361650fb0ba5 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-down.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-down.pug
@@ -15,4 +15,4 @@
     limitations under the License.
 
 mixin ignite-form-field-down()
-    i.tipField.fa.fa-arrow-down(ignite-form-field-down ng-click="$ctrl.down()")&attributes(attributes)
+    i.tipField.fa.fa-arrow-down(ignite-form-field-down ng-click='$ctrl.down()')&attributes(attributes)
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug
similarity index 80%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug
index 33af6d1d8785a..d7bd0fd2943ea 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug
@@ -17,18 +17,18 @@
 mixin ignite-form-field-dropdown(label, model, name, disabled, required, multiple, placeholder, placeholderEmpty, options, tip)
     mixin form-field-input()
         button.select-toggle.form-control(
-            id='{{ #{name} }}Input'
-            name='{{ #{name} }}'
+            id=`{{ ${name} }}Input`
+            name=`{{ ${name} }}`
 
-            data-placeholder=placeholderEmpty ? '{{ #{options}.length > 0 ? "#{placeholder}" : "#{placeholderEmpty}" }}' : placeholder
+            data-placeholder=placeholderEmpty ? `{{ ${options}.length > 0 ? '${placeholder}' : '${placeholderEmpty}' }}` : placeholder
             
             data-ng-model=model
 
-            data-ng-required=required && '#{required}'
-            data-ng-disabled=disabled && '#{disabled}' || '!#{options}.length'
+            data-ng-required=required && `${required}`
+            data-ng-disabled=disabled && `${disabled}` || `!${options}.length`
 
             bs-select
-            bs-options='item.value as item.label for item in #{options}'
+            bs-options=`item.value as item.label for item in ${options}`
 
             data-multiple=multiple ? '1' : false
             data-container='body > .wrapper'
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
similarity index 83%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
index ac1f0802683c4..c70e7a3b02e6b 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
@@ -15,13 +15,13 @@
     limitations under the License.
 
 mixin form-field-feedback(name, error, message)
-    -var __field = form + '[' + name + ']'
-    -var __error = __field + '.$error.' + error
-    -var __pristine = __field + '.$pristine'
+    -var __field = `${form}[${name}]`
+    -var __error = `${__field}.$error.${error}`
+    -var __pristine = `${__field}.$pristine`
 
     i.fa.fa-exclamation-triangle.form-field-feedback(
-        ng-if='!#{__pristine} && #{__error}'
-        name='{{ #{name} }}'
+        ng-if=`!${__pristine} && ${__error}`
+        name=`{{ ${name} }}`
 
         bs-tooltip
         data-title=message
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-label.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-label.pug
similarity index 88%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-label.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-label.pug
index d6aef8192cc4b..d0275c94ee010 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-label.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-label.pug
@@ -16,8 +16,8 @@
 
 mixin ignite-form-field__label(label, name, required)
     label.ignite-form-field__label(
-        id='{{ #{name} }}Label'
-        for='{{ #{name} }}Input'
-        class="{{ #{required} ? 'required' : '' }}"
+        id=`{{ ${name} }}Label`
+        for=`{{ ${name} }}Input`
+        class=`{{ ${required} ? 'required' : '' }}`
     )
         span !{label}
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-number.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-number.pug
similarity index 91%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-number.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-number.pug
index 58b0dcd654291..dd41849d1e7ce 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-number.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-number.pug
@@ -17,8 +17,8 @@
 mixin ignite-form-field-number(label, model, name, disabled, required, placeholder, min, max, step, tip)
     mixin form-field-input()
         input.form-control(
-            id='{{ #{name} }}Input'
-            name='{{ #{name} }}'
+            id=`{{ ${name} }}Input`
+            name=`{{ ${name} }}`
             placeholder=placeholder
             type='number'
 
@@ -28,8 +28,8 @@ mixin ignite-form-field-number(label, model, name, disabled, required, placehold
 
             data-ng-model=model
 
-            data-ng-required=required && '#{required}'
-            data-ng-disabled=disabled && '#{disabled}'
+            data-ng-required=required && `${required}`
+            data-ng-disabled=disabled && `${disabled}`
             data-ng-focus='tableReset()'
 
             data-ignite-form-panel-field=''
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-password.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
similarity index 91%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-password.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
index e5e7bc83f3114..a12982f4fb6e8 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-password.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
@@ -16,15 +16,15 @@
 
 mixin ignite-form-field-password-input(name, model, disabled, required, placeholder)
     input.form-control(
-        id='{{ #{name} }}Input'
-        name='{{ #{name} }}'
+        id=`{{ ${name} }}Input`
+        name=`{{ ${name} }}`
         placeholder=placeholder
         type='password'
 
         data-ng-model=model
 
-        data-ng-required=required && '#{required}'
-        data-ng-disabled=disabled && '#{disabled}'
+        data-ng-required=required && `${required}`
+        data-ng-disabled=disabled && `${disabled}`
         data-ng-focus='tableReset()'
 
         data-ignite-form-panel-field=''
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
similarity index 86%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-text.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
index 1f93d3b8ede50..76ea6e609d6e1 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
@@ -16,15 +16,15 @@
 
 mixin ignite-form-field-input(name, model, disabled, required, placeholder)
     input.form-control(
-        id='{{ #{name} }}Input'
-        name='{{ #{name} }}'
+        id=`{{ ${name} }}Input`
+        name=`{{ ${name} }}`
         placeholder=placeholder
         type='text'
 
         data-ng-model=model
 
-        data-ng-required=required && '#{required}'
-        data-ng-disabled=disabled && '#{disabled}'
+        data-ng-required=required && `${required}`
+        data-ng-disabled=disabled && `${disabled}`
         data-ng-focus='tableReset()'
 
         data-ignite-form-panel-field=''
@@ -32,15 +32,15 @@ mixin ignite-form-field-input(name, model, disabled, required, placeholder)
 
 mixin ignite-form-field-url-input(name, model, disabled, required, placeholder)
     input.form-control(
-        id='{{ #{name} }}Input'
-        name='{{ #{name} }}'
+        id=`{{ ${name} }}Input`
+        name=`{{ ${name} }}`
         placeholder=placeholder
         type='url'
 
         data-ng-model=model
 
-        data-ng-required=required && '#{required}'
-        data-ng-disabled=disabled && '#{disabled}'
+        data-ng-required=required && `${required}`
+        data-ng-disabled=disabled && `${disabled}`
         data-ng-focus='tableReset()'
 
         data-ignite-form-panel-field=''
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-up.jade b/modules/web-console/frontend/app/helpers/jade/form/form-field-up.pug
similarity index 95%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-up.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-field-up.pug
index c66cd0e6021c4..5b6520166e111 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-up.jade
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-up.pug
@@ -15,4 +15,4 @@
     limitations under the License.
 
 mixin ignite-form-field-up()
-    i.tipField.fa.fa-arrow-up.ng-scope(ignite-form-field-up ng-click="$ctrl.up()")&attributes(attributes)
+    i.tipField.fa.fa-arrow-up.ng-scope(ignite-form-field-up ng-click='$ctrl.up()')&attributes(attributes)
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-group.jade b/modules/web-console/frontend/app/helpers/jade/form/form-group.pug
similarity index 100%
rename from modules/web-console/frontend/app/helpers/jade/form/form-group.jade
rename to modules/web-console/frontend/app/helpers/jade/form/form-group.pug
diff --git a/modules/web-console/frontend/app/helpers/jade/mixins.jade b/modules/web-console/frontend/app/helpers/jade/mixins.pug
similarity index 91%
rename from modules/web-console/frontend/app/helpers/jade/mixins.jade
rename to modules/web-console/frontend/app/helpers/jade/mixins.pug
index 6ca41f645c68c..db175a2146ef1 100644
--- a/modules/web-console/frontend/app/helpers/jade/mixins.jade
+++ b/modules/web-console/frontend/app/helpers/jade/mixins.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include ./form.jade
+include ./form
 
 //- Mixin for advanced options toggle.
 mixin advanced-options-toggle(click, cond, showMessage, hideMessage)
     .advanced-options
-        i.fa(ng-click='#{click}' ng-class='#{cond} ? "fa-chevron-circle-down" : "fa-chevron-circle-right"')
-        a(ng-click=click) {{#{cond} ? '#{hideMessage}' : '#{showMessage}'}}
+        i.fa(ng-click=`${click}` ng-class=`${cond} ? 'fa-chevron-circle-down' : 'fa-chevron-circle-right'`)
+        a(ng-click=click) {{ #{cond} ? '#{hideMessage}' : '#{showMessage}' }}
 
 //- Mixin for advanced options toggle with default settings.
 mixin advanced-options-toggle-default
@@ -28,14 +28,14 @@ mixin advanced-options-toggle-default
 
 //- Mixin for main table on screen with list of items.
 mixin main-table(title, rows, focusId, click, rowTemplate, searchField)
-    .padding-bottom-dflt(ng-show='#{rows} && #{rows}.length > 0')
-        table.links(st-table='displayedRows' st-safe-src='#{rows}')
+    .padding-bottom-dflt(ng-show=`${rows} && ${rows}.length > 0`)
+        table.links(st-table='displayedRows' st-safe-src=`${rows}`)
             thead
                 tr
                     th
                         lable.labelHeader.labelFormField #{title}:
                         .col-sm-3.pull-right(style='padding: 0')
-                            input.form-control(type='text' st-search='#{searchField}' placeholder='Filter #{title}...')
+                            input.form-control(type='text' st-search=`${searchField}` placeholder=`Filter ${title}...`)
             tbody
                 tr
                     td
@@ -55,9 +55,9 @@ mixin save-remove-clone-undo-buttons(objectName)
 
     div(ng-show='contentVisible()' style='display: inline-block;')
         .panel-tip-container(ng-hide='!backupItem || backupItem._id')
-            a.btn.btn-primary(ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && saveItem()' bs-tooltip='' data-title='{{saveBtnTipText(ui.inputForm.$dirty, "#{objectName}")}}' data-placement='bottom' data-trigger='hover') Save
+            a.btn.btn-primary(ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && saveItem()' bs-tooltip='' data-title=`{{saveBtnTipText(ui.inputForm.$dirty, '${objectName}')}}` data-placement='bottom' data-trigger='hover') Save
         .panel-tip-container(ng-show='backupItem._id')
-            a.btn.btn-primary(id='save-item' ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && saveItem()' bs-tooltip='' data-title='{{saveBtnTipText(ui.inputForm.$dirty, "#{objectName}")}}' data-placement='bottom' data-trigger='hover') Save
+            a.btn.btn-primary(id='save-item' ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && saveItem()' bs-tooltip='' data-title=`{{saveBtnTipText(ui.inputForm.$dirty, '${objectName}')}}` data-placement='bottom' data-trigger='hover') Save
         .panel-tip-container(ng-show='backupItem._id')
             a.btn.btn-primary(id='clone-item' ng-click='cloneItem()' bs-tooltip=cloneTip data-placement='bottom' data-trigger='hover') Clone
         .btn-group.panel-tip-container(ng-show='backupItem._id')
@@ -71,7 +71,7 @@ mixin save-remove-clone-undo-buttons(objectName)
 mixin error-feedback(visible, error, errorMessage, name)
     i.fa.fa-exclamation-triangle.form-control-feedback(
         ng-if=visible
-        bs-tooltip='"#{errorMessage}"'
+        bs-tooltip=`'${errorMessage}'`
         ignite-error=error
         ignite-error-message=errorMessage
         name=name
@@ -120,7 +120,7 @@ mixin java-class-autofocus-placholder(lbl, model, name, enabled, required, autof
         data-java-keywords='true'
         data-java-built-in-class='true'
         data-ignite-form-field-input-autofocus=autofocus
-        data-validation-active=validationActive ? '{{ #{validationActive} }}' : '"always"'
+        data-validation-active=validationActive ? `{{ ${validationActive} }}` : `'always'`
     )
         if  block
             block
@@ -150,7 +150,7 @@ mixin java-class-typeahead(lbl, model, name, options, enabled, required, placeho
         data-java-identifier='true'
         data-java-package-specified='allow-built-in'
         data-java-keywords='true'
-        data-validation-active=validationActive ? '{{ #{validationActive} }}' : '"always"'
+        data-validation-active=validationActive ? `{{ ${validationActive} }}` : `'always'`
     )
         +form-field-feedback(name, 'javaKeywords', errLbl + ' could not contains reserved Java keyword!')
         +form-field-feedback(name, 'javaPackageSpecified', errLbl + ' does not have package specified!')
@@ -266,12 +266,12 @@ mixin dropdown-multiple(lbl, model, name, enabled, placeholder, placeholderEmpty
 //- Mixin for table text field.
 mixin table-text-field(name, model, items, valid, save, placeholder, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         if block
@@ -289,12 +289,12 @@ mixin table-text-field(name, model, items, valid, save, placeholder, newItem)
 //- Mixin for table java class field.
 mixin table-java-class-field(lbl, name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         +form-field-feedback(name, 'javaBuiltInClass', lbl + ' should not be the Java built-in class!')
@@ -322,12 +322,12 @@ mixin table-java-class-field(lbl, name, model, items, valid, save, newItem)
 //- Mixin for table java package field.
 mixin table-java-package-field(name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         +form-field-feedback(name, 'javaKeywords', 'Package name could not contains reserved Java keyword!')
@@ -351,12 +351,12 @@ mixin table-java-package-field(name, model, items, valid, save, newItem)
 //- Mixin for table java package field.
 mixin table-url-field(name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         if block
@@ -374,12 +374,12 @@ mixin table-url-field(name, model, items, valid, save, newItem)
 //- Mixin for table address field.
 mixin table-address-field(name, model, items, valid, save, newItem, portRange)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         +ipaddress-feedback(name)
@@ -405,12 +405,12 @@ mixin table-address-field(name, model, items, valid, save, newItem, portRange)
 //- Mixin for table UUID field.
 mixin table-uuid-field(name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         if block
@@ -433,7 +433,7 @@ mixin table-save-button(valid, save, newItem)
 
     i.fa.fa-floppy-o.form-field-save(
         ng-show=valid
-        ng-click='!(#{valid}) || (#{save}); !(#{valid}) || (#{reset});'
+        ng-click=`!(${valid}) || (${save}); !(${valid}) || (${reset});`
         bs-tooltip
         data-title='Click icon or press [Enter] to save item'
     )
@@ -441,10 +441,10 @@ mixin table-save-button(valid, save, newItem)
 //- Mixin for table remove button.
 mixin table-remove-conditional-button(items, show, tip, row)
     i.tipField.fa.fa-remove(
-        ng-hide='!#{show} || field.edit'
+        ng-hide=`!${show} || field.edit`
         bs-tooltip
         data-title=tip
-        ng-click='#{items}.splice(#{items}.indexOf(#{row}), 1)'
+        ng-click=`${items}.splice(${items}.indexOf(${row}), 1)`
     )
 
 //- Mixin for table remove button.
@@ -551,11 +551,11 @@ mixin btn-remove-cond(cond, click, tip)
 
 //- LEGACY mixin for LEGACY pair values tables.
 mixin table-pair-edit(tbl, prefix, keyPlaceholder, valPlaceholder, keyJavaBuiltInTypes, valueJavaBuiltInTypes, focusId, index, divider)
-    -var keyModel = tbl + '.' + prefix + 'Key'
-    -var valModel = tbl +'.' + prefix + 'Value'
+    -var keyModel = `${tbl}.${prefix}Key`
+    -var valModel = `${tbl}.${prefix}Value`
 
-    -var keyFocusId = prefix + 'Key' + focusId
-    -var valFocusId = prefix + 'Value' + focusId
+    -var keyFocusId = `${prefix}Key${focusId}`
+    -var valFocusId = `${prefix}Value${focusId}`
 
     .col-xs-6.col-sm-6.col-md-6
         .fieldSep !{divider}
@@ -601,9 +601,9 @@ mixin dialect(lbl, model, name, required, tipTitle, genericDialectName, placehol
 //- Mixin for show/hide links.
 mixin showHideLink(name, text)
     span(ng-init='__ = {};')
-        a.customize(ng-show='__.#{name}' ng-click='__.#{name} = false') Hide #{text}
-        a.customize(ng-hide='__.#{name}' ng-click='__.#{name} = true; ui.loadPanel("#{name}");') Show #{text}
-        div(ng-if='ui.isPanelLoaded("#{name}")')
-            .panel-details(ng-show='__.#{name}')
+        a.customize(ng-show=`__.${name}` ng-click=`__.${name} = false`) Hide #{text}
+        a.customize(ng-hide=`__.${name}` ng-click=`__.${name} = true; ui.loadPanel('${name}');`) Show #{text}
+        div(ng-if=`ui.isPanelLoaded('${name}')`)
+            .panel-details(ng-show=`__.${name}`)
                 if block
                     block
diff --git a/modules/web-console/frontend/app/modules/agent/agent.module.js b/modules/web-console/frontend/app/modules/agent/agent.module.js
index 7ac39d10f3594..c0e92d5d2db99 100644
--- a/modules/web-console/frontend/app/modules/agent/agent.module.js
+++ b/modules/web-console/frontend/app/modules/agent/agent.module.js
@@ -18,6 +18,8 @@
 import angular from 'angular';
 import io from 'socket.io-client'; // eslint-disable-line no-unused-vars
 
+import templateUrl from 'views/templates/agent-download.tpl.pug';
+
 const maskNull = (val) => _.isEmpty(val) ? 'null' : val;
 
 class IgniteAgentMonitor {
@@ -35,7 +37,7 @@ class IgniteAgentMonitor {
         // Pre-fetch modal dialogs.
         this._downloadAgentModal = $modal({
             scope: this._scope,
-            templateUrl: '/templates/agent-download.html',
+            templateUrl,
             show: false,
             backdrop: 'static',
             keyboard: false
diff --git a/modules/web-console/frontend/app/modules/branding/header-logo.directive.js b/modules/web-console/frontend/app/modules/branding/header-logo.directive.js
index 423de9c1c9965..231411b818601 100644
--- a/modules/web-console/frontend/app/modules/branding/header-logo.directive.js
+++ b/modules/web-console/frontend/app/modules/branding/header-logo.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './header-logo.jade';
+import template from './header-logo.pug';
 
 export default ['igniteHeaderLogo', ['IgniteBranding', (branding) => {
     function controller() {
@@ -26,7 +26,7 @@ export default ['igniteHeaderLogo', ['IgniteBranding', (branding) => {
 
     return {
         restrict: 'E',
-        templateUrl,
+        template,
         controller,
         controllerAs: 'logo',
         replace: true
diff --git a/modules/web-console/frontend/app/modules/branding/header-logo.jade b/modules/web-console/frontend/app/modules/branding/header-logo.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/branding/header-logo.jade
rename to modules/web-console/frontend/app/modules/branding/header-logo.pug
diff --git a/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js b/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js
index 2f02446c77ffb..dce7d556353ef 100644
--- a/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js
+++ b/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './powered-by-apache.jade';
+import template from './powered-by-apache.pug';
 
 export default ['ignitePoweredByApache', ['IgniteBranding', (branding) => {
     function controller() {
@@ -26,7 +26,7 @@ export default ['ignitePoweredByApache', ['IgniteBranding', (branding) => {
 
     return {
         restrict: 'E',
-        templateUrl,
+        template,
         controller,
         controllerAs: 'poweredBy',
         replace: true
diff --git a/modules/web-console/frontend/app/modules/branding/powered-by-apache.jade b/modules/web-console/frontend/app/modules/branding/powered-by-apache.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/branding/powered-by-apache.jade
rename to modules/web-console/frontend/app/modules/branding/powered-by-apache.pug
diff --git a/modules/web-console/frontend/app/modules/demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js
index bd759df552c23..740ab30fec4e1 100644
--- a/modules/web-console/frontend/app/modules/demo/Demo.module.js
+++ b/modules/web-console/frontend/app/modules/demo/Demo.module.js
@@ -18,6 +18,7 @@
 import angular from 'angular';
 
 import DEMO_INFO from 'app/data/demo-info.json';
+import templateUrl from 'views/templates/demo-info.tpl.pug';
 
 angular
 .module('ignite-console.demo', [
@@ -129,7 +130,7 @@ angular
     }
 
     const dialog = $modal({
-        templateUrl: '/templates/demo-info.html',
+        templateUrl,
         scope,
         placement: 'center',
         show: false,
diff --git a/modules/web-console/frontend/app/modules/dialog/dialog.factory.js b/modules/web-console/frontend/app/modules/dialog/dialog.factory.js
index e15891f347ed7..2ac891793dc4d 100644
--- a/modules/web-console/frontend/app/modules/dialog/dialog.factory.js
+++ b/modules/web-console/frontend/app/modules/dialog/dialog.factory.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import templateUrl from './dialog.jade';
+import templateUrl from './dialog.tpl.pug';
 
 export default ['IgniteDialog', ['$modal', ($modal) => {
     const defaults = {
diff --git a/modules/web-console/frontend/app/modules/dialog/dialog.jade b/modules/web-console/frontend/app/modules/dialog/dialog.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/dialog/dialog.jade
rename to modules/web-console/frontend/app/modules/dialog/dialog.tpl.pug
diff --git a/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js b/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js
index cf9f561bd837c..f4c834cfcc0ab 100644
--- a/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js
+++ b/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js
@@ -19,6 +19,7 @@ import angular from 'angular';
 
 // Getting started pages.
 import PAGES from 'app/data/getting-started.json';
+import templateUrl from 'views/templates/getting-started.tpl.pug';
 
 angular
     .module('ignite-console.getting-started', [])
@@ -77,7 +78,7 @@ angular
             _fillPage();
         };
 
-        const dialog = $modal({templateUrl: '/templates/getting-started.html', scope, placement: 'center', show: false, backdrop: 'static'});
+        const dialog = $modal({ templateUrl, scope, placement: 'center', show: false, backdrop: 'static'});
 
         scope.close = () => {
             try {
diff --git a/modules/web-console/frontend/app/modules/loading/loading.directive.js b/modules/web-console/frontend/app/modules/loading/loading.directive.js
index 064b4c28b2566..8a4ca9f5fab8c 100644
--- a/modules/web-console/frontend/app/modules/loading/loading.directive.js
+++ b/modules/web-console/frontend/app/modules/loading/loading.directive.js
@@ -15,12 +15,12 @@
  * limitations under the License.
  */
 
-import templateUrl from './loading.jade';
-import './loading.css';
+import template from './loading.pug';
+import './loading.scss';
 
-export default ['igniteLoading', ['IgniteLoading', '$templateCache', '$compile', (Loading, $templateCache, $compile) => {
+export default ['igniteLoading', ['IgniteLoading', '$compile', (Loading, $compile) => {
     const link = (scope, element) => {
-        const compiledTemplate = $compile($templateCache.get(templateUrl));
+        const compiledTemplate = $compile(template);
 
         const build = () => {
             scope.position = scope.position || 'middle';
diff --git a/modules/web-console/frontend/app/modules/loading/loading.jade b/modules/web-console/frontend/app/modules/loading/loading.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/loading/loading.jade
rename to modules/web-console/frontend/app/modules/loading/loading.pug
diff --git a/modules/web-console/frontend/app/modules/loading/loading.css b/modules/web-console/frontend/app/modules/loading/loading.scss
similarity index 100%
rename from modules/web-console/frontend/app/modules/loading/loading.css
rename to modules/web-console/frontend/app/modules/loading/loading.scss
diff --git a/modules/web-console/frontend/app/modules/nodes/Nodes.service.js b/modules/web-console/frontend/app/modules/nodes/Nodes.service.js
index b320ae4461a31..4ca1d45f6aacf 100644
--- a/modules/web-console/frontend/app/modules/nodes/Nodes.service.js
+++ b/modules/web-console/frontend/app/modules/nodes/Nodes.service.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import nodesDialogTemplate from './nodes-dialog.jade';
+import nodesDialogTplUrl from './nodes-dialog.tpl.pug';
 
 const DEFAULT_OPTIONS = {
     grid: {
@@ -41,7 +41,7 @@ class Nodes {
         options.target = cacheName;
 
         const modalInstance = $modal({
-            templateUrl: nodesDialogTemplate,
+            templateUrl: nodesDialogTplUrl,
             show: true,
             resolve: {
                 nodes: () => nodes || [],
diff --git a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.jade b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/nodes/nodes-dialog.jade
rename to modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
diff --git a/modules/web-console/frontend/app/modules/sql/notebook.controller.js b/modules/web-console/frontend/app/modules/sql/notebook.controller.js
index f10a4d005da4e..252dee65c844b 100644
--- a/modules/web-console/frontend/app/modules/sql/notebook.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/notebook.controller.js
@@ -15,11 +15,13 @@
  * limitations under the License.
  */
 
+import templateUrl from 'views/sql/notebook-new.tpl.pug';
+
 // Controller that load notebooks in navigation bar .
 export default ['$scope', '$modal', '$state', 'IgniteMessages', 'IgniteNotebook',
     (scope, $modal, $state, Messages, Notebook) => {
         // Pre-fetch modal dialogs.
-        const nameModal = $modal({scope, templateUrl: '/sql/notebook-new.html', show: false});
+        const nameModal = $modal({scope, templateUrl, show: false});
 
         scope.create = (name) => {
             return Notebook.create(name)
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 4e972ef88ab6f..e2ead13ec7c41 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -15,6 +15,12 @@
  * limitations under the License.
  */
 
+import paragraphRateTemplateUrl from 'views/sql/paragraph-rate.tpl.pug';
+import cacheMetadataTemplateUrl from 'views/sql/cache-metadata.tpl.pug';
+import chartSettingsTemplateUrl from 'views/sql/chart-settings.tpl.pug';
+import showQueryTemplateUrl from 'views/templates/message.tpl.pug';
+
+
 // Time line X axis descriptor.
 const TIME_LINE = {value: -1, type: 'java.sql.Date', label: 'TIME_LINE'};
 
@@ -188,6 +194,13 @@ class Paragraph {
 // Controller for SQL notebook screen.
 export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData',
     function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData) {
+        const $ctrl = this;
+
+        // Define template urls.
+        $ctrl.paragraphRateTemplateUrl = paragraphRateTemplateUrl;
+        $ctrl.cacheMetadataTemplateUrl = cacheMetadataTemplateUrl;
+        $ctrl.chartSettingsTemplateUrl = chartSettingsTemplateUrl;
+
         let stopTopology = null;
 
         const _tryStopRefresh = function(paragraph) {
@@ -1737,7 +1750,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 }
 
                 // Show a basic modal from a controller
-                $modal({scope, template: '/templates/message.html', placement: 'center', show: true});
+                $modal({scope, templateUrl: showQueryTemplateUrl, placement: 'center', show: true});
             }
         };
     }
diff --git a/modules/web-console/frontend/app/modules/sql/sql.module.js b/modules/web-console/frontend/app/modules/sql/sql.module.js
index 5875961f0a896..670e4f45090d9 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.module.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.module.js
@@ -20,7 +20,9 @@ import angular from 'angular';
 import NotebookData from './Notebook.data';
 import Notebook from './Notebook.service';
 import notebook from './notebook.controller';
-import sql from './sql.controller';
+import controller from './sql.controller';
+
+import sqlTplUrl from 'app/../views/sql/sql.tpl.pug';
 
 angular.module('ignite-console.sql', [
     'ui.router'
@@ -32,11 +34,13 @@ angular.module('ignite-console.sql', [
                 .state('base.sql', {
                     url: '/queries',
                     abstract: true,
-                    template: '<ui-view></ui-view>'
+                    template: '<ui-view></ui-view>',
+                    controller,
+                    controllerAs: '$ctrl'
                 })
                 .state('base.sql.notebook', {
                     url: '/notebook/{noteId}',
-                    templateUrl: '/sql/sql.html',
+                    templateUrl: sqlTplUrl,
                     onEnter: AclRoute.checkAccess('query'),
                     metaTags: {
                         title: 'Query notebook'
@@ -44,7 +48,7 @@ angular.module('ignite-console.sql', [
                 })
                 .state('base.sql.demo', {
                     url: '/demo',
-                    templateUrl: '/sql/sql.html',
+                    templateUrl: sqlTplUrl,
                     onEnter: AclRoute.checkAccess('query'),
                     metaTags: {
                         title: 'SQL demo'
@@ -54,5 +58,4 @@ angular.module('ignite-console.sql', [
     )
     .service('IgniteNotebookData', NotebookData)
     .service('IgniteNotebook', Notebook)
-    .controller('notebookController', notebook)
-    .controller('sqlController', sql);
+    .controller('notebookController', notebook);
diff --git a/modules/web-console/frontend/app/modules/states/admin.state.js b/modules/web-console/frontend/app/modules/states/admin.state.js
index 35c6fbbfeeab0..93a38ddd8c697 100644
--- a/modules/web-console/frontend/app/modules/states/admin.state.js
+++ b/modules/web-console/frontend/app/modules/states/admin.state.js
@@ -17,6 +17,8 @@
 
 import angular from 'angular';
 
+import templateUrl from 'views/settings/admin.tpl.pug';
+
 angular
 .module('ignite-console.states.admin', [
     'ui.router'
@@ -26,7 +28,7 @@ angular
     $stateProvider
     .state('settings.admin', {
         url: '/admin',
-        templateUrl: '/settings/admin.html',
+        templateUrl,
         onEnter: AclRoute.checkAccess('admin_page'),
         metaTags: {
             title: 'Admin panel'
diff --git a/modules/web-console/frontend/app/modules/states/configuration.state.js b/modules/web-console/frontend/app/modules/states/configuration.state.js
index 61dca13f9633b..a609d3a414409 100644
--- a/modules/web-console/frontend/app/modules/states/configuration.state.js
+++ b/modules/web-console/frontend/app/modules/states/configuration.state.js
@@ -26,6 +26,14 @@ import ConfigurationResource from './configuration/Configuration.resource';
 import summaryTabs from './configuration/summary/summary-tabs.directive';
 import IgniteSummaryZipper from './configuration/summary/summary-zipper.service';
 
+import sidebarTpl from 'views/configuration/sidebar.tpl.pug';
+import clustersTpl from 'views/configuration/clusters.tpl.pug';
+import cachesTpl from 'views/configuration/caches.tpl.pug';
+import domainsTpl from 'views/configuration/domains.tpl.pug';
+import igfsTpl from 'views/configuration/igfs.tpl.pug';
+import summaryTpl from 'views/configuration/summary.tpl.pug';
+import summaryTabsTemplateUrl from 'views/configuration/summary-tabs.pug';
+
 angular.module('ignite-console.states.configuration', ['ui.router'])
     .directive(...previewPanel)
     // Summary screen
@@ -33,18 +41,21 @@ angular.module('ignite-console.states.configuration', ['ui.router'])
     // Services.
     .service('IgniteSummaryZipper', IgniteSummaryZipper)
     .service('IgniteConfigurationResource', ConfigurationResource)
+    .run(['$templateCache', ($templateCache) => {
+        $templateCache.put('summary-tabs.html', summaryTabsTemplateUrl);
+    }])
     // Configure state provider.
     .config(['$stateProvider', 'AclRouteProvider', ($stateProvider, AclRoute) => {
         // Setup the states.
         $stateProvider
             .state('base.configuration', {
                 url: '/configuration',
-                templateUrl: '/configuration/sidebar.html',
+                templateUrl: sidebarTpl,
                 abstract: true
             })
             .state('base.configuration.clusters', {
                 url: '/clusters',
-                templateUrl: '/configuration/clusters.html',
+                templateUrl: clustersTpl,
                 onEnter: AclRoute.checkAccess('configuration'),
                 params: {
                     linkId: null
@@ -55,7 +66,7 @@ angular.module('ignite-console.states.configuration', ['ui.router'])
             })
             .state('base.configuration.caches', {
                 url: '/caches',
-                templateUrl: '/configuration/caches.html',
+                templateUrl: cachesTpl,
                 onEnter: AclRoute.checkAccess('configuration'),
                 params: {
                     linkId: null
@@ -66,7 +77,7 @@ angular.module('ignite-console.states.configuration', ['ui.router'])
             })
             .state('base.configuration.domains', {
                 url: '/domains',
-                templateUrl: '/configuration/domains.html',
+                templateUrl: domainsTpl,
                 onEnter: AclRoute.checkAccess('configuration'),
                 params: {
                     linkId: null
@@ -77,7 +88,7 @@ angular.module('ignite-console.states.configuration', ['ui.router'])
             })
             .state('base.configuration.igfs', {
                 url: '/igfs',
-                templateUrl: '/configuration/igfs.html',
+                templateUrl: igfsTpl,
                 onEnter: AclRoute.checkAccess('configuration'),
                 params: {
                     linkId: null
@@ -88,7 +99,7 @@ angular.module('ignite-console.states.configuration', ['ui.router'])
             })
             .state('base.configuration.summary', {
                 url: '/summary',
-                templateUrl: '/configuration/summary.html',
+                templateUrl: summaryTpl,
                 onEnter: AclRoute.checkAccess('configuration'),
                 controller: ConfigurationSummaryCtrl,
                 controllerAs: 'ctrl',
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug
similarity index 77%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug
index 3c4746bbca794..70b90bda0c4cd 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'affinity'
 -var model = 'backupItem'
@@ -28,18 +28,18 @@ include /app/helpers/jade/mixins.jade
 -var fairPartitionsRequired = fairAff + ' && ' + affModel + '.Fair.affinityBackupFilter'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Affinity Collocation
         ignite-form-field-tooltip.tipLabel
             | Collocate data with data to improve performance and scalability of your application#[br]
             | #[a(href="http://apacheignite.gridgain.org/docs/affinity-collocation" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Function:', affModel + '.kind', '"AffinityKind"', 'true', 'Default',
+                    +dropdown('Function:', `${affModel}.kind`, '"AffinityKind"', 'true', 'Default',
                         '[\
                             {value: "Rendezvous", label: "Rendezvous"},\
                             {value: "Fair", label: "Fair"},\
@@ -55,25 +55,25 @@ include /app/helpers/jade/mixins.jade
                         </ul>')
                 .panel-details(ng-if=rendezvousAff)
                     .details-row
-                        +number-required('Partitions', affModel + '.Rendezvous.partitions', '"RendPartitions"', 'true', rendPartitionsRequired, '1024', '1', 'Number of partitions')
+                        +number-required('Partitions', `${affModel}.Rendezvous.partitions`, '"RendPartitions"', 'true', rendPartitionsRequired, '1024', '1', 'Number of partitions')
                     .details-row
-                        +java-class('Backup filter', affModel + '.Rendezvous.affinityBackupFilter', '"RendAffinityBackupFilter"', 'true', 'false',
+                        +java-class('Backup filter', `${affModel}.Rendezvous.affinityBackupFilter`, '"RendAffinityBackupFilter"', 'true', 'false',
                             'Backups will be selected from all nodes that pass this filter')
                     .details-row
-                        +checkbox('Exclude neighbors', affModel + '.Rendezvous.excludeNeighbors', '"RendExcludeNeighbors"',
+                        +checkbox('Exclude neighbors', `${affModel}.Rendezvous.excludeNeighbors`, '"RendExcludeNeighbors"',
                             'Exclude same - host - neighbors from being backups of each other and specified number of backups')
                 .panel-details(ng-if=fairAff)
                     .details-row
-                        +number-required('Partitions', affModel + '.Fair.partitions', '"FairPartitions"', 'true', fairPartitionsRequired, '256', '1', 'Number of partitions')
+                        +number-required('Partitions', `${affModel}.Fair.partitions`, '"FairPartitions"', 'true', fairPartitionsRequired, '256', '1', 'Number of partitions')
                     .details-row
-                        +java-class('Backup filter', affModel + '.Fair.affinityBackupFilter', '"FairAffinityBackupFilter"', 'true', 'false',
+                        +java-class('Backup filter', `${affModel}.Fair.affinityBackupFilter`, '"FairAffinityBackupFilter"', 'true', 'false',
                             'Backups will be selected from all nodes that pass this filter')
                     .details-row
-                        +checkbox('Exclude neighbors', affModel + '.Fair.excludeNeighbors', '"FairExcludeNeighbors"',
+                        +checkbox('Exclude neighbors', `${affModel}.Fair.excludeNeighbors`, '"FairExcludeNeighbors"',
                             'Exclude same - host - neighbors from being backups of each other and specified number of backups')
                 .panel-details(ng-if=customAff)
                     .details-row
-                        +java-class('Class name:', affModel + '.Custom.className', '"AffCustomClassName"', 'true', customAff,
+                        +java-class('Class name:', `${affModel}.Custom.className`, '"AffCustomClassName"', 'true', customAff,
                             'Custom key affinity function implementation class name')
                 .settings-row
                     +java-class('Mapper:', model + '.affinityMapper', '"AffMapCustomClassName"', 'true', 'false',
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.pug
similarity index 81%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.pug
index e3d1a81589ced..cd5bcc821fca5 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/client-near-cache.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'clientNearCache'
 -var model = 'backupItem.clientNearConfiguration'
 
 .panel.panel-default(ng-form=form novalidate ng-show='backupItem.cacheMode === "PARTITIONED"')
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Client near cache
         ignite-form-field-tooltip.tipLabel
@@ -28,18 +28,18 @@ include /app/helpers/jade/mixins.jade
             | Near cache is a small local cache that stores most recently or most frequently accessed data#[br]
             | Should be used in case when it is impossible to send computations to remote nodes
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var enabled = model + '.clientNearCacheEnabled'
+                -var enabled = `${model}.clientNearCacheEnabled`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"clientNacheEnabled"', 'Flag indicating whether to configure near cache')
                 .settings-row
-                    +number('Start size:', model + '.nearStartSize', '"clientNearStartSize"', enabled, '375000', '0',
+                    +number('Start size:', `${model}.nearStartSize`, '"clientNearStartSize"', enabled, '375000', '0',
                         'Initial cache size for near cache which will be used to pre-create internal hash table after start')
                 .settings-row
-                    +evictionPolicy(model + '.nearEvictionPolicy', '"clientNearCacheEvictionPolicy"', enabled, 'false',
+                    +evictionPolicy(`${model}.nearEvictionPolicy`, '"clientNearCacheEvictionPolicy"', enabled, 'false',
                         'Near cache eviction policy\
                         <ul>\
                             <li>Least Recently Used (LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.pug
similarity index 79%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.pug
index ffcd568852731..6458fbb5d3603 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.pug
@@ -14,31 +14,31 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'concurrency'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Concurrency control
         ignite-form-field-tooltip.tipLabel
             | Cache concurrent asynchronous operations settings
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Max async operations:', model + '.maxConcurrentAsyncOperations', '"maxConcurrentAsyncOperations"', 'true', '500', '0',
+                    +number('Max async operations:', `${model}.maxConcurrentAsyncOperations`, '"maxConcurrentAsyncOperations"', 'true', '500', '0',
                         'Maximum number of allowed concurrent asynchronous operations<br/>\
                         If <b>0</b> then number of concurrent asynchronous operations is unlimited')
                 .settings-row
-                    +number('Default lock timeout:', model + '.defaultLockTimeout', '"defaultLockTimeout"', 'true', '0', '0',
+                    +number('Default lock timeout:', `${model}.defaultLockTimeout`, '"defaultLockTimeout"', 'true', '0', '0',
                         'Default lock acquisition timeout in milliseconds<br/>\
                         If <b>0</b> then lock acquisition will never timeout')
-                .settings-row(ng-hide='#{model}.atomicityMode === "TRANSACTIONAL"')
-                    +dropdown('Entry versioning:', model + '.atomicWriteOrderMode', '"atomicWriteOrderMode"', 'true', 'Choose versioning',
+                .settings-row(ng-hide=`${model}.atomicityMode === 'TRANSACTIONAL'`)
+                    +dropdown('Entry versioning:', `${model}.atomicWriteOrderMode`, '"atomicWriteOrderMode"', 'true', 'Choose versioning',
                         '[\
                             {value: "CLOCK", label: "CLOCK"},\
                             {value: "PRIMARY", label: "PRIMARY"}\
@@ -49,7 +49,7 @@ include /app/helpers/jade/mixins.jade
                             <li>PRIMARY - in this mode version is assigned only on primary node. This means that sender will only send write request to primary node, which in turn will assign write version and forward it to backups</li>\
                         </ul>')
                 .settings-row
-                    +dropdown('Write synchronization mode:', model + '.writeSynchronizationMode', '"writeSynchronizationMode"', 'true', 'PRIMARY_SYNC',
+                    +dropdown('Write synchronization mode:', `${model}.writeSynchronizationMode`, '"writeSynchronizationMode"', 'true', 'PRIMARY_SYNC',
                         '[\
                             {value: "FULL_SYNC", label: "FULL_SYNC"},\
                             {value: "FULL_ASYNC", label: "FULL_ASYNC"},\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/general.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/general.pug
similarity index 77%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/general.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/general.pug
index 14f3ab45e5a63..89676f20e30e1 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/general.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/general.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var model = 'backupItem'
@@ -31,17 +31,17 @@ include /app/helpers/jade/mixins.jade
         .panel-body
             .col-sm-6
                 .settings-row
-                    +text('Name:', model + '.name', '"cacheName"', 'true', 'Input name', 'Cache name')
+                    +text('Name:', `${model}.name`, '"cacheName"', 'true', 'Input name', 'Cache name')
                 .settings-row
                     +clusters(model, 'Associate clusters with the current cache')
                 .settings-row
                     +dropdown-multiple('<span>Domain models:</span><a ui-sref="base.configuration.domains({linkId: linkId()})"> (add)</a>',
-                        model + '.domains', '"domains"', true, 'Choose domain models', 'No valid domain models configured', 'domains',
+                        `${model}.domains`, '"domains"', true, 'Choose domain models', 'No valid domain models configured', 'domains',
                         'Select domain models to describe types in cache')
                 .settings-row
-                    +cacheMode('Mode:', model + '.cacheMode', '"cacheMode"', 'PARTITIONED')
+                    +cacheMode('Mode:', `${model}.cacheMode`, '"cacheMode"', 'PARTITIONED')
                 .settings-row
-                    +dropdown('Atomicity:', model + '.atomicityMode', '"atomicityMode"', 'true', 'ATOMIC',
+                    +dropdown('Atomicity:', `${model}.atomicityMode`, '"atomicityMode"', 'true', 'ATOMIC',
                         '[\
                             {value: "ATOMIC", label: "ATOMIC"},\
                             {value: "TRANSACTIONAL", label: "TRANSACTIONAL"}\
@@ -51,18 +51,18 @@ include /app/helpers/jade/mixins.jade
                             <li>ATOMIC - in this mode distributed transactions and distributed locking are not supported</li>\
                             <li>TRANSACTIONAL - in this mode specified fully ACID-compliant transactional cache behavior</li>\
                         </ul>')
-                .settings-row(data-ng-show='#{model}.cacheMode === "PARTITIONED"')
-                    +number('Backups:', model + '.backups', '"backups"', 'true', '0', '0', 'Number of nodes used to back up single partition for partitioned cache')
-                .settings-row(data-ng-show='#{model}.cacheMode === "PARTITIONED" && #{model}.backups')
-                    +checkbox('Read from backup', model + '.readFromBackup', '"readFromBackup"',
+                .settings-row(data-ng-show=`${model}.cacheMode === 'PARTITIONED'`)
+                    +number('Backups:', `${model}.backups`, '"backups"', 'true', '0', '0', 'Number of nodes used to back up single partition for partitioned cache')
+                .settings-row(data-ng-show=`${model}.cacheMode === 'PARTITIONED' && ${model}.backups`)
+                    +checkbox('Read from backup', `${model}.readFromBackup`, '"readFromBackup"',
                         'Flag indicating whether data can be read from backup<br/>\
                         If not set then always get data from primary node (never from backup)')
                 .settings-row
-                    +checkbox('Copy on read', model + '.copyOnRead', '"copyOnRead"',
+                    +checkbox('Copy on read', `${model}.copyOnRead`, '"copyOnRead"',
                         'Flag indicating whether copy of the value stored in cache should be created for cache operation implying return value<br/>\
                         Also if this flag is set copies are created for values passed to CacheInterceptor and to CacheEntryProcessor')
-                .settings-row(ng-show='#{model}.cacheMode === "PARTITIONED" && #{model}.atomicityMode === "TRANSACTIONAL"')
-                    +checkbox('Invalidate near cache', model + '.invalidate', '"invalidate"',
+                .settings-row(ng-show=`${model}.cacheMode === 'PARTITIONED' && ${model}.atomicityMode === 'TRANSACTIONAL'`)
+                    +checkbox('Invalidate near cache', `${model}.invalidate`, '"invalidate"',
                         'Invalidation flag for near cache entries in transaction<br/>\
                         If set then values will be invalidated (nullified) upon commit in near cache')
             .col-sm-6
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/memory.pug
similarity index 86%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/memory.pug
index e8dfb3a15a1fc..38482a78f8f06 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/memory.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/memory.pug
@@ -14,24 +14,24 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'memory'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Memory
         ignite-form-field-tooltip.tipLabel
             | Cache memory settings#[br]
             | #[a(href="https://apacheignite.readme.io/docs/off-heap-memory" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Mode:', model + '.memoryMode', '"memoryMode"', 'true', 'ONHEAP_TIERED',
+                    +dropdown('Mode:', `${model}.memoryMode`, '"memoryMode"', 'true', 'ONHEAP_TIERED',
                         '[\
                             {value: "ONHEAP_TIERED", label: "ONHEAP_TIERED"},\
                             {value: "OFFHEAP_TIERED", label: "OFFHEAP_TIERED"},\
@@ -61,9 +61,8 @@ include /app/helpers/jade/mixins.jade
                                 Note that in this mode entries can be evicted only to swap\
                             </li>\
                         </ul>')
-                .settings-row(ng-show=model + '.memoryMode !== "OFFHEAP_VALUES"')
-                    +dropdown-required('Off-heap memory:', model + '.offHeapMode', '"offHeapMode"', 'true',
-                        model + '.memoryMode === "OFFHEAP_TIERED"',
+                .settings-row(ng-show=`${model}.memoryMode !== 'OFFHEAP_VALUES'`)
+                    +dropdown-required('Off-heap memory:', `${model}.offHeapMode`, '"offHeapMode"', 'true', `${model}.memoryMode === 'OFFHEAP_TIERED'`,
                         'Disabled',
                         '[\
                             {value: -1, label: "Disabled"},\
@@ -76,16 +75,16 @@ include /app/helpers/jade/mixins.jade
                             <li>Limited - Off-heap storage has limited size</li>\
                             <li>Unlimited - Off-heap storage grow infinitely (it is up to user to properly add and remove entries from cache to ensure that off-heap storage does not grow infinitely)</li>\
                         </ul>')
-                .settings-row(ng-if=model + '.offHeapMode === 1 && ' + model + '.memoryMode !== "OFFHEAP_VALUES"')
-                    +number-required('Off-heap memory max size:', model + '.offHeapMaxMemory', '"offHeapMaxMemory"', 'true',
-                        model + '.offHeapMode === 1', 'Enter off-heap memory size', '1',
+                .settings-row(ng-if=`${model}.offHeapMode === 1 && ${model}.memoryMode !== 'OFFHEAP_VALUES'`)
+                    +number-required('Off-heap memory max size:', `${model}.offHeapMaxMemory`, '"offHeapMaxMemory"', 'true',
+                        `${model}.offHeapMode === 1`, 'Enter off-heap memory size', '1',
                         'Maximum amount of memory available to off-heap storage in bytes')
                 .settings-row
                     -var onHeapTired = model + '.memoryMode === "ONHEAP_TIERED"'
                     -var swapEnabled = model + '.swapEnabled'
                     -var offHeapMaxMemory = model + '.offHeapMaxMemory'
 
-                    +evictionPolicy(model + '.evictionPolicy', '"evictionPolicy"', 'true',
+                    +evictionPolicy(`${model}.evictionPolicy`, '"evictionPolicy"', 'true',
                         onHeapTired  + ' && (' + swapEnabled + '|| _.isNumber(' + offHeapMaxMemory + ') &&' + offHeapMaxMemory + ' >= 0)',
                         'Optional cache eviction policy<br/>\
                         Must be set for entries to be evicted from on-heap to off-heap or swap\
@@ -95,7 +94,7 @@ include /app/helpers/jade/mixins.jade
                             <li>SORTED - Eviction policy which will select the minimum cache entry for eviction</li>\
                         </ul>')
                 .settings-row
-                    +number('Start size:', model + '.startSize', '"startSize"', 'true', '1500000', '0',
+                    +number('Start size:', `${model}.startSize`, '"startSize"', 'true', '1500000', '0',
                         'In terms of size and capacity, Ignite internal cache map acts exactly like a normal Java HashMap: it has some initial capacity\
                         (which is pretty small by default), which doubles as data arrives. The process of internal cache map resizing is CPU-intensive\
                         and time-consuming, and if you load a huge dataset into cache (which is a normal use case), the map will have to resize a lot of times.\
@@ -104,6 +103,6 @@ include /app/helpers/jade/mixins.jade
                         For example, if you expect to load 10 million entries into cache, you can set this property to 10 000 000.\
                         This will save you from cache internal map resizes.')
                 .settings-row
-                    +checkbox('Swap enabled', model + '.swapEnabled', '"swapEnabled"', 'Flag indicating whether swap storage is enabled or not for this cache')
+                    +checkbox('Swap enabled', `${model}.swapEnabled`, '"swapEnabled"', 'Flag indicating whether swap storage is enabled or not for this cache')
             .col-sm-6
                 +preview-xml-java(model, 'cacheMemory')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.pug
similarity index 76%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.pug
index 56f7e648c8cc3..499dd2de230b2 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'clientNearCache'
 -var model = 'backupItem'
 
-.panel.panel-default(ng-form=form novalidate ng-show='backupItem.cacheMode === "PARTITIONED"')
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+.panel.panel-default(ng-form=form novalidate ng-show=`${model}.cacheMode === 'PARTITIONED'`)
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Near cache on client node
         ignite-form-field-tooltip.tipLabel
@@ -28,19 +28,19 @@ include /app/helpers/jade/mixins.jade
             | Near cache is a small local cache that stores most recently or most frequently accessed data#[br]
             | Should be used in case when it is impossible to send computations to remote nodes
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var nearCfg = model + '.clientNearConfiguration'
-                -var enabled = nearCfg + '.enabled'
+                -var nearCfg = `${model}.clientNearConfiguration`
+                -var enabled = `${nearCfg}.enabled`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"clientNearEnabled"', 'Flag indicating whether to configure near cache')
                 .settings-row
-                    +number('Start size:', nearCfg + '.nearStartSize', '"clientNearStartSize"', enabled, '375000', '0',
+                    +number('Start size:', `${nearCfg}.nearStartSize`, '"clientNearStartSize"', enabled, '375000', '0',
                         'Initial cache size for near cache which will be used to pre-create internal hash table after start')
                 .settings-row
-                    +evictionPolicy(nearCfg + '.nearEvictionPolicy', '"clientNearCacheEvictionPolicy"', enabled, 'false',
+                    +evictionPolicy(`${nearCfg}.nearEvictionPolic`, '"clientNearCacheEvictionPolicy"', enabled, 'false',
                         'Near cache eviction policy\
                         <ul>\
                             <li>Least Recently Used (LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.pug
similarity index 77%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.pug
index 9895281486ef3..2efe43a9e849c 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'serverNearCache'
 -var model = 'backupItem'
 
-.panel.panel-default(ng-form=form novalidate ng-show='#{model}.cacheMode === "PARTITIONED"')
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+.panel.panel-default(ng-form=form novalidate ng-show=`${model}.cacheMode === 'PARTITIONED'`)
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Near cache on server node
         ignite-form-field-tooltip.tipLabel
@@ -29,19 +29,19 @@ include /app/helpers/jade/mixins.jade
             | Should be used in case when it is impossible to send computations to remote nodes#[br]
             | #[a(href="https://apacheignite.readme.io/docs/near-caches" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var nearCfg = model + '.nearConfiguration'
-                -var enabled = nearCfg + '.enabled'
+                -var nearCfg = `${model}.nearConfiguration`
+                -var enabled = `${nearCfg}.enabled`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"nearCacheEnabled"', 'Flag indicating whether to configure near cache')
                 .settings-row
-                    +number('Start size:', nearCfg + '.nearStartSize', '"nearStartSize"', enabled, '375000', '0',
+                    +number('Start size:', `${nearCfg}.nearStartSize`, '"nearStartSize"', enabled, '375000', '0',
                         'Initial cache size for near cache which will be used to pre-create internal hash table after start')
                 .settings-row
-                    +evictionPolicy(model + '.nearConfiguration.nearEvictionPolicy', '"nearCacheEvictionPolicy"', enabled, 'false',
+                    +evictionPolicy(`${model}.nearConfiguration.nearEvictionPolicy`, '"nearCacheEvictionPolicy"', enabled, 'false',
                         'Near cache eviction policy\
                         <ul>\
                             <li>Least Recently Used (LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.pug
similarity index 71%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.pug
index bcac5adcaf995..6715dcd259aa6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.pug
@@ -14,20 +14,20 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'nodeFilter'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label(id='nodeFilter-title') Node filter
         ignite-form-field-tooltip.tipLabel
             | Determines on what nodes the cache should be started
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     -var nodeFilter = model + '.nodeFilter';
@@ -42,18 +42,18 @@ include /app/helpers/jade/mixins.jade
                         'Node filter variant'
                     )
                 .settings-row(ng-show=nodeFilterKind)
-                    div(ng-show='#{nodeFilterKind} === "IGFS"')
-                        -var igfsNodeFilter = nodeFilter + '.IGFS'
-                        -var required = nodeFilterKind + ' === "IGFS"'
+                    div(ng-show=`${nodeFilterKind} === 'IGFS'`)
+                        -var igfsNodeFilter = `${nodeFilter}.IGFS`
+                        -var required = `${nodeFilterKind} === 'IGFS'`
 
                         //(lbl, model, name, enabled, required, placeholder, options, tip)
-                        +dropdown-required-empty('IGFS:', igfsNodeFilter + '.igfs', '"igfsNodeFilter"', 'true', required,
+                        +dropdown-required-empty('IGFS:', `${igfsNodeFilter}.igfs`, '"igfsNodeFilter"', 'true', required,
                             'Choose IGFS', 'No IGFS configured', 'igfss', 'Select IGFS to filter nodes')
-                    div(ng-show='#{nodeFilterKind} === "Custom"')
-                        -var customNodeFilter = nodeFilter + '.Custom'
-                        -var required = nodeFilterKind + ' === "Custom"'
+                    div(ng-show=`${nodeFilterKind} === 'Custom'`)
+                        -var customNodeFilter = `${nodeFilter}.Custom`
+                        -var required = `${nodeFilterKind} === 'Custom'`
 
-                        +java-class('Class name:', customNodeFilter + '.className', '"customNodeFilter"',
+                        +java-class('Class name:', `${customNodeFilter}.className`, '"customNodeFilter"',
                             'true', required, 'Class name of custom node filter implementation', required)
             .col-sm-6
                 +preview-xml-java(model, 'cacheNodeFilter', 'igfss')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/query.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/query.pug
similarity index 76%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/query.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/query.pug
index cfbaf12c57a8a..5bb515aba7dc1 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/query.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/query.pug
@@ -14,24 +14,24 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'query'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Queries & Indexing
         ignite-form-field-tooltip.tipLabel
             | Cache queries settings#[br]
             | #[a(href="https://apacheignite.readme.io/docs/sql-queries" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +text('SQL schema name:', model + '.sqlSchema', '"sqlSchema"', 'false', 'Input schema name',
+                    +text('SQL schema name:', `${model}.sqlSchema`, '"sqlSchema"', 'false', 'Input schema name',
                         'Specify any custom name to be used as SQL schema for current cache. This name will correspond to SQL ANSI-99 standard.\
                         Nonquoted identifiers are not case sensitive. Quoted identifiers are case sensitive.\
                         When SQL schema is not specified, quoted cache name should used instead.<br/>\
@@ -47,19 +47,19 @@ include /app/helpers/jade/mixins.jade
                             </li>\
                         </ul>')
                 .settings-row
-                    +number('On-heap cache for off-heap indexes:', model + '.sqlOnheapRowCacheSize', '"sqlOnheapRowCacheSize"', 'true', '10240', '1',
+                    +number('On-heap cache for off-heap indexes:', `${model}.sqlOnheapRowCacheSize`, '"sqlOnheapRowCacheSize"', 'true', '10240', '1',
                         'Number of SQL rows which will be cached onheap to avoid deserialization on each SQL index access')
                 .settings-row
-                    +number('Long query timeout:', model + '.longQueryWarningTimeout', '"longQueryWarningTimeout"', 'true', '3000', '0',
+                    +number('Long query timeout:', `${model}.longQueryWarningTimeout`, '"longQueryWarningTimeout"', 'true', '3000', '0',
                         'Timeout in milliseconds after which long query warning will be printed')
                 .settings-row
-                    +number('History size:', model + '.queryDetailMetricsSize', '"queryDetailMetricsSize"', 'true', '0', '0',
+                    +number('History size:', `${model}.queryDetailMetricsSize`, '"queryDetailMetricsSize"', 'true', '0', '0',
                         'Size of queries detail metrics that will be stored in memory for monitoring purposes')
                 .settings-row
                     -var form = 'querySqlFunctionClasses';
-                    -var sqlFunctionClasses = model + '.sqlFunctionClasses';
+                    -var sqlFunctionClasses = `${model}.sqlFunctionClasses`;
 
-                    +ignite-form-group(ng-form=form ng-model=sqlFunctionClasses)
+                    +ignite-form-group(ng-form=form ng-model=`${sqlFunctionClasses}`)
                         ignite-form-field-label
                             | SQL functions
                         ignite-form-group-tooltip
@@ -69,20 +69,20 @@ include /app/helpers/jade/mixins.jade
 
                         -var uniqueTip = 'SQL function with such class name already exists!'
 
-                        .group-content(ng-if='#{sqlFunctionClasses}.length')
+                        .group-content(ng-if=`${sqlFunctionClasses}.length`)
                             -var model = 'obj.model';
                             -var name = '"edit" + $index'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = sqlFunctionClasses + '[$index] = ' + model
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${sqlFunctionClasses}[$index] = ${model}`
 
-                            div(ng-repeat='model in #{sqlFunctionClasses} track by $index' ng-init='obj = {}')
+                            div(ng-repeat=`model in ${sqlFunctionClasses} track by $index` ng-init='obj = {}')
                                 label.col-xs-12.col-sm-12.col-md-12
                                     .indexField
                                         | {{ $index+1 }})
                                     +table-remove-button(sqlFunctionClasses, 'Remove user-defined function')
 
                                     span(ng-hide='field.edit')
-                                        a.labelFormField(ng-click='field.edit = true; #{model} = model;') {{ model }}
+                                        a.labelFormField(ng-click=`field.edit = true; ${model} = model;`) {{ model }}
                                     span(ng-if='field.edit')
                                         +table-java-class-field('SQL function', name, model, sqlFunctionClasses, valid, save, false)
                                             +table-save-button(valid, save, false)
@@ -91,8 +91,8 @@ include /app/helpers/jade/mixins.jade
                         .group-content(ng-repeat='field in group.add')
                             -var model = 'new';
                             -var name = '"new"'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = sqlFunctionClasses + '.push(' + model + ')'
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${sqlFunctionClasses}.push(${model})`
 
                             div
                                 label.col-xs-12.col-sm-12.col-md-12
@@ -100,13 +100,13 @@ include /app/helpers/jade/mixins.jade
                                         +table-save-button(valid, save, true)
                                         +unique-feedback(name, uniqueTip)
 
-                        .group-content-empty(ng-if='!(#{sqlFunctionClasses}.length) && !group.add.length')
+                        .group-content-empty(ng-if=`!(${sqlFunctionClasses}.length) && !group.add.length`)
                             | Not defined
                 .settings-row
-                    +checkbox('Snapshotable index', model + '.snapshotableIndex', '"snapshotableIndex"',
+                    +checkbox('Snapshotable index', `${model}.snapshotableIndex`, '"snapshotableIndex"',
                         'Flag indicating whether SQL indexes should support snapshots')
                 .settings-row
-                    +checkbox('Escape table and filed names', model + '.sqlEscapeAll', '"sqlEscapeAll"',
+                    +checkbox('Escape table and filed names', `${model}.sqlEscapeAll`, '"sqlEscapeAll"',
                         'If enabled than all schema, table and field names will be escaped with double quotes (for example: "tableName"."fieldName").<br/>\
                         This enforces case sensitivity for field names and also allows having special characters in table and field names.<br/>\
                         Escaped names will be used for creation internal structures in Ignite SQL engine.')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.pug
similarity index 75%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.pug
index d8ef3adb62239..9850d171c9ce7 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.pug
@@ -14,24 +14,24 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'rebalance'
 -var model = 'backupItem'
 
-.panel.panel-default(ng-form=form novalidate ng-hide='#{model}.cacheMode === "LOCAL"')
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+.panel.panel-default(ng-form=form novalidate ng-hide=`${model}.cacheMode === "LOCAL"`)
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Rebalance
         ignite-form-field-tooltip.tipLabel
             | Cache rebalance settings#[br]
             | #[a(href="https://apacheignite.readme.io/docs/rebalancing" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Mode:', model + '.rebalanceMode', '"rebalanceMode"', 'true', 'ASYNC',
+                    +dropdown('Mode:', `${model}.rebalanceMode`, '"rebalanceMode"', 'true', 'ASYNC',
                         '[\
                             {value: "SYNC", label: "SYNC"},\
                             {value: "ASYNC", label: "ASYNC"},\
@@ -44,23 +44,23 @@ include /app/helpers/jade/mixins.jade
                             <li>None - in this mode no rebalancing will take place which means that caches will be either loaded on demand from persistent store whenever data is accessed, or will be populated explicitly</li>\
                         </ul>')
                     .settings-row
-                        +number('Batch size:', model + '.rebalanceBatchSize', '"rebalanceBatchSize"', 'true', '512 * 1024', '1',
+                        +number('Batch size:', `${model}.rebalanceBatchSize`, '"rebalanceBatchSize"', 'true', '512 * 1024', '1',
                             'Size (in bytes) to be loaded within a single rebalance message<br/>\
                             Rebalancing algorithm will split total data set on every node into multiple batches prior to sending data')
                     .settings-row
-                        +number('Batches prefetch count:', model + '.rebalanceBatchesPrefetchCount', '"rebalanceBatchesPrefetchCount"', 'true', '2', '1',
+                        +number('Batches prefetch count:', `${model}.rebalanceBatchesPrefetchCount`, '"rebalanceBatchesPrefetchCount"', 'true', '2', '1',
                             'Number of batches generated by supply node at rebalancing start')
                     .settings-row
-                        +number('Order:', model + '.rebalanceOrder', '"rebalanceOrder"', 'true', '0', Number.MIN_SAFE_INTEGER,
+                        +number('Order:', `${model}.rebalanceOrder`, '"rebalanceOrder"', 'true', '0', Number.MIN_SAFE_INTEGER,
                             'If cache rebalance order is positive, rebalancing for this cache will be started only when rebalancing for all caches with smaller rebalance order (except caches with rebalance order 0) will be completed')
                     .settings-row
-                        +number('Delay:', model + '.rebalanceDelay', '"rebalanceDelay"', 'true', '0', '0',
+                        +number('Delay:', `${model}.rebalanceDelay`, '"rebalanceDelay"', 'true', '0', '0',
                             'Delay in milliseconds upon a node joining or leaving topology (or crash) after which rebalancing should be started automatically')
                     .settings-row
-                        +number('Timeout:', model + '.rebalanceTimeout', '"rebalanceTimeout"', 'true', '10000', '0',
+                        +number('Timeout:', `${model}.rebalanceTimeout`, '"rebalanceTimeout"', 'true', '10000', '0',
                             'Rebalance timeout in milliseconds')
                     .settings-row
-                        +number('Throttle:', model + '.rebalanceThrottle', '"rebalanceThrottle"', 'true', '0', '0',
+                        +number('Throttle:', `${model}.rebalanceThrottle`, '"rebalanceThrottle"', 'true', '0', '0',
                             'Time in milliseconds to wait between rebalance messages to avoid overloading of CPU or network')
             .col-sm-6
                 +preview-xml-java(model, 'cacheRebalance')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/statistics.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/statistics.pug
similarity index 76%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/statistics.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/statistics.pug
index 9f7a3465b29fb..1eeea84f77656 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/statistics.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/statistics.pug
@@ -14,25 +14,25 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'statistics'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Statistics
         ignite-form-field-tooltip.tipLabel
             | Cache statistics and management settings
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +checkbox('Statistics enabled', model + '.statisticsEnabled', '"statisticsEnabled"', 'Flag indicating whether statistics gathering is enabled on this cache')
+                    +checkbox('Statistics enabled', `${model}.statisticsEnabled`, '"statisticsEnabled"', 'Flag indicating whether statistics gathering is enabled on this cache')
                 .settings-row
-                    +checkbox('Management enabled', model + '.managementEnabled', '"managementEnabled"',
+                    +checkbox('Management enabled', `${model}.managementEnabled`, '"managementEnabled"',
                     'Flag indicating whether management is enabled on this cache<br/>\
                     If enabled the CacheMXBean for each cache is registered in the platform MBean server')
             .col-sm-6
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/store.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/store.pug
similarity index 72%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/store.jade
rename to modules/web-console/frontend/app/modules/states/configuration/caches/store.pug
index ea350f2394ef7..f09bc0b2cca43 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/store.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/caches/store.pug
@@ -14,19 +14,19 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'store'
 -var model = 'backupItem'
 
 mixin hibernateField(name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = valid + ' && (' + save + '); ' + valid + ' && ' + resetOnEnter + ';'
+    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
 
     -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
 
     -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = valid + ' && ( ' + save + '); ' + resetOnBlur + ';'
+    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
 
     div(ignite-on-focus-out=onBlur)
         if block
@@ -43,19 +43,19 @@ mixin hibernateField(name, model, items, valid, save, newItem)
             )
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Store
         ignite-form-field-tooltip.tipLabel
             | Cache store settings#[br]
             | #[a(href="https://apacheignite.readme.io/docs/persistent-store" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    -var storeFactory = model + '.cacheStoreFactory';
-                    -var storeFactoryKind = storeFactory + '.kind';
+                    -var storeFactory = `${model}.cacheStoreFactory`;
+                    -var storeFactoryKind = `${storeFactory}.kind`;
 
                     +dropdown('Store factory:', storeFactoryKind, '"cacheStoreFactory"', 'true', 'Not set',
                         '[\
@@ -75,44 +75,44 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                         a.customize(ng-show='__.expanded' ng-click='__.expanded = false') Hide settings
                         a.customize(ng-hide='__.expanded' ng-click='__.expanded = true') Show settings
                         .panel-details(ng-show='__.expanded')
-                            div(ng-show='#{storeFactoryKind} === "CacheJdbcPojoStoreFactory"')
-                                -var pojoStoreFactory = storeFactory + '.CacheJdbcPojoStoreFactory'
-                                -var required = storeFactoryKind + ' === "CacheJdbcPojoStoreFactory"'
+                            div(ng-show=`${storeFactoryKind} === 'CacheJdbcPojoStoreFactory'`)
+                                -var pojoStoreFactory = `${storeFactory}.CacheJdbcPojoStoreFactory`
+                                -var required = `${storeFactoryKind} === 'CacheJdbcPojoStoreFactory'`
 
                                 .details-row
-                                    +text('Data source bean name:', pojoStoreFactory + '.dataSourceBean',
+                                    +text('Data source bean name:', `${pojoStoreFactory}.dataSourceBean`,
                                         '"pojoDataSourceBean"', required, 'Input bean name',
                                         'Name of the data source bean in Spring context')
                                 .details-row
-                                    +dialect('Dialect:', pojoStoreFactory + '.dialect', '"pojoDialect"', required,
+                                    +dialect('Dialect:', `${pojoStoreFactory}.dialect`, '"pojoDialect"', required,
                                         'Dialect of SQL implemented by a particular RDBMS:', 'Generic JDBC dialect',
                                         'Choose JDBC dialect')
                                 .details-row
-                                    +number('Batch size:', pojoStoreFactory + '.batchSize', '"pojoBatchSize"', true, '512', '1',
+                                    +number('Batch size:', `${pojoStoreFactory}.batchSize`, '"pojoBatchSize"', true, '512', '1',
                                         'Maximum batch size for writeAll and deleteAll operations')
                                 .details-row
-                                    +number('Thread count:', pojoStoreFactory + '.maximumPoolSize', '"pojoMaximumPoolSize"', true, 'availableProcessors', '1',
+                                    +number('Thread count:', `${pojoStoreFactory}.maximumPoolSize`, '"pojoMaximumPoolSize"', true, 'availableProcessors', '1',
                                         'Maximum workers thread count.<br/>\
                                         These threads are responsible for load cache.')
                                 .details-row
-                                    +number('Maximum write attempts:', pojoStoreFactory + '.maximumWriteAttempts', '"pojoMaximumWriteAttempts"', true, '2', '0',
+                                    +number('Maximum write attempts:', `${pojoStoreFactory}.maximumWriteAttempts`, '"pojoMaximumWriteAttempts"', true, '2', '0',
                                         'Maximum write attempts in case of database error')
                                 .details-row
-                                    +number('Parallel load threshold:', pojoStoreFactory + '.parallelLoadCacheMinimumThreshold', '"pojoParallelLoadCacheMinimumThreshold"', true, '512', '0',
+                                    +number('Parallel load threshold:', `${pojoStoreFactory}.parallelLoadCacheMinimumThreshold`, '"pojoParallelLoadCacheMinimumThreshold"', true, '512', '0',
                                         'Parallel load cache minimum threshold.<br/>\
                                         If <b>0</b> then load sequentially.')
                                 .details-row
-                                    +java-class('Hasher', pojoStoreFactory + '.hasher', '"pojoHasher"', 'true', 'false', 'Hash calculator', required)
+                                    +java-class('Hasher', `${pojoStoreFactory}.hasher`, '"pojoHasher"', 'true', 'false', 'Hash calculator', required)
                                 .details-row
-                                    +java-class('Transformer', pojoStoreFactory + '.transformer', '"pojoTransformer"', 'true', 'false', 'Types transformer', required)
+                                    +java-class('Transformer', `${pojoStoreFactory}.transformer`, '"pojoTransformer"', 'true', 'false', 'Types transformer', required)
                                 .details-row
-                                    +checkbox('Escape table and filed names', pojoStoreFactory + '.sqlEscapeAll', '"sqlEscapeAll"',
+                                    +checkbox('Escape table and filed names', `${pojoStoreFactory}.sqlEscapeAll`, '"sqlEscapeAll"',
                                         'If enabled than all schema, table and field names will be escaped with double quotes (for example: "tableName"."fieldName").<br/>\
                                         This enforces case sensitivity for field names and also allows having special characters in table and field names.<br/>\
                                         Escaped names will be used for CacheJdbcPojoStore internal SQL queries.')
-                            div(ng-show='#{storeFactoryKind} === "CacheJdbcBlobStoreFactory"')
-                                -var blobStoreFactory = storeFactory + '.CacheJdbcBlobStoreFactory'
-                                -var blobStoreFactoryVia = blobStoreFactory + '.connectVia'
+                            div(ng-show=`${storeFactoryKind} === 'CacheJdbcBlobStoreFactory'`)
+                                -var blobStoreFactory = `${storeFactory}.CacheJdbcBlobStoreFactory`
+                                -var blobStoreFactoryVia = `${blobStoreFactory}.connectVia`
 
                                 .details-row
                                     +dropdown('Connect via:', blobStoreFactoryVia, '"connectVia"', 'true', 'Choose connection method',
@@ -125,51 +125,51 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                                             <li>JDBC URL, for example: jdbc:h2:mem:myDatabase</li>\
                                             <li>Configured data source</li>\
                                         </ul>')
-                                div(ng-show='#{blobStoreFactoryVia} === "URL"')
-                                    -var required = storeFactoryKind + ' === "CacheJdbcBlobStoreFactory" && ' + blobStoreFactoryVia + ' === "URL"'
+                                div(ng-show=`${blobStoreFactoryVia} === 'URL'`)
+                                    -var required = `${storeFactoryKind} === 'CacheJdbcBlobStoreFactory' && ${blobStoreFactoryVia} === 'URL'`
 
                                     .details-row
-                                        +text('Connection URL:', blobStoreFactory + '.connectionUrl', '"connectionUrl"', required, 'Input URL',
+                                        +text('Connection URL:', `${blobStoreFactory}.connectionUrl`, '"connectionUrl"', required, 'Input URL',
                                             'URL for database access, for example: jdbc:h2:mem:myDatabase')
                                     .details-row
-                                        +text('User:', blobStoreFactory + '.user', '"user"', required, 'Input user name', 'User name for database access')
+                                        +text('User:', `${blobStoreFactory}.user`, '"user"', required, 'Input user name', 'User name for database access')
                                     .details-row
                                         label Note, password will be generated as stub
-                                div(ng-show='#{blobStoreFactoryVia} !== "URL"')
-                                    -var required = storeFactoryKind + ' === "CacheJdbcBlobStoreFactory" && ' + blobStoreFactoryVia + '!== "URL"'
+                                div(ng-show=`${blobStoreFactoryVia} !== 'URL'`)
+                                    -var required = `${storeFactoryKind} === 'CacheJdbcBlobStoreFactory' && ${blobStoreFactoryVia} !== 'URL'`
 
                                     .details-row
-                                        +text('Data source bean name:', blobStoreFactory + '.dataSourceBean', '"blobDataSourceBean"', required, 'Input bean name',
+                                        +text('Data source bean name:', `${blobStoreFactory}.dataSourceBean`, '"blobDataSourceBean"', required, 'Input bean name',
                                             'Name of the data source bean in Spring context')
                                     .details-row
-                                        +dialect('Database:', blobStoreFactory + '.dialect', '"blobDialect"', required, 'Supported databases:', 'Generic database', 'Choose database')
+                                        +dialect('Database:', `${blobStoreFactory}.dialect`, '"blobDialect"', required, 'Supported databases:', 'Generic database', 'Choose database')
                                 .details-row
-                                    +checkbox('Init schema', blobStoreFactory + '.initSchema', '"initSchema"',
+                                    +checkbox('Init schema', `${blobStoreFactory}.initSchema`, '"initSchema"',
                                         'Flag indicating whether DB schema should be initialized by Ignite (default behaviour) or was explicitly created by user')
                                 .details-row
-                                    +text('Create query:', blobStoreFactory + '.createTableQuery', '"createTableQuery"', 'false', 'SQL for table creation',
+                                    +text('Create query:', `${blobStoreFactory}.createTableQuery`, '"createTableQuery"', 'false', 'SQL for table creation',
                                         'Query for table creation in underlying database<br/>\
                                         Default value: create table if not exists ENTRIES (key binary primary key, val binary)')
                                 .details-row
-                                    +text('Load query:', blobStoreFactory + '.loadQuery', '"loadQuery"', 'false', 'SQL for load entry',
+                                    +text('Load query:', `${blobStoreFactory}.loadQuery`, '"loadQuery"', 'false', 'SQL for load entry',
                                         'Query for entry load from underlying database<br/>\
                                         Default value: select * from ENTRIES where key=?')
                                 .details-row
-                                    +text('Insert query:', blobStoreFactory + '.insertQuery', '"insertQuery"', 'false', 'SQL for insert entry',
+                                    +text('Insert query:', `${blobStoreFactory}.insertQuery`, '"insertQuery"', 'false', 'SQL for insert entry',
                                         'Query for insert entry into underlying database<br/>\
                                         Default value: insert into ENTRIES (key, val) values (?, ?)')
                                 .details-row
-                                    +text('Update query:', blobStoreFactory + '.updateQuery', '"updateQuery"', 'false', 'SQL for update entry',
+                                    +text('Update query:', `${blobStoreFactory}.updateQuery`, '"updateQuery"', 'false', 'SQL for update entry',
                                         'Query for update entry in underlying database<br/>\
                                         Default value: update ENTRIES set val=? where key=?')
                                 .details-row
-                                    +text('Delete query:', blobStoreFactory + '.deleteQuery', '"deleteQuery"', 'false', 'SQL for delete entry',
+                                    +text('Delete query:', `${blobStoreFactory}.deleteQuery`, '"deleteQuery"', 'false', 'SQL for delete entry',
                                         'Query for delete entry from underlying database<br/>\
                                         Default value: delete from ENTRIES where key=?')
 
-                            div(ng-show='#{storeFactoryKind} === "CacheHibernateBlobStoreFactory"')
-                                -var hibernateStoreFactory = storeFactory + '.CacheHibernateBlobStoreFactory'
-                                -var hibernateProperties = hibernateStoreFactory + '.hibernateProperties'
+                            div(ng-show=`${storeFactoryKind} === 'CacheHibernateBlobStoreFactory'`)
+                                -var hibernateStoreFactory = `${storeFactory}.CacheHibernateBlobStoreFactory`
+                                -var hibernateProperties = `${hibernateStoreFactory}.hibernateProperties`
 
                                 .details-row
                                     +ignite-form-group(ng-form=form ng-model=hibernateProperties)
@@ -184,12 +184,12 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                                         -var tipUnique = 'Property with such key already exists!'
                                         -var tipPropertySpecified = 'Property should be present in format key=value!'
 
-                                        .group-content-empty(ng-if='!((#{hibernateProperties} && #{hibernateProperties}.length > 0) || tableNewItemActive(hibernatePropsTbl))')
+                                        .group-content-empty(ng-if=`!((${hibernateProperties} && ${hibernateProperties}.length > 0) || tableNewItemActive(hibernatePropsTbl))`)
                                             | Not defined
-                                        .group-content(ng-show='(#{hibernateProperties} && #{hibernateProperties}.length > 0) || tableNewItemActive(hibernatePropsTbl)')
+                                        .group-content(ng-show=`(${hibernateProperties} && ${hibernateProperties}.length > 0) || tableNewItemActive(hibernatePropsTbl)`)
                                             table.links-edit(id='hibernateProps' st-table=hibernateProperties)
                                                 tbody
-                                                    tr(ng-repeat='item in #{hibernateProperties}')
+                                                    tr(ng-repeat=`item in ${hibernateProperties}`)
                                                         td.col-sm-12(ng-hide='tableEditing(hibernatePropsTbl, $index)')
                                                             a.labelFormField(ng-click='tableStartEdit(backupItem, hibernatePropsTbl, $index)') {{item.name}} = {{item.value}}
                                                             +btn-remove('tableRemove(backupItem, hibernatePropsTbl, $index)', '"Remove Property"')
@@ -202,10 +202,10 @@ mixin hibernateField(name, model, items, valid, save, newItem)
 
 
                 .settings-row
-                    +checkbox('Keep binary in store', model + '.storeKeepBinary', '"storeKeepBinary"',
+                    +checkbox('Keep binary in store', `${model}.storeKeepBinary`, '"storeKeepBinary"',
                         'Flag indicating that CacheStore implementation is working with binary objects instead of Java objects')
                 .settings-row
-                    +checkbox('Load previous value', model + '.loadPreviousValue', '"loadPreviousValue"',
+                    +checkbox('Load previous value', `${model}.loadPreviousValue`, '"loadPreviousValue"',
                         'Flag indicating whether value should be loaded from store if it is not in the cache for following cache operations: \
                         <ul> \
                             <li>IgniteCache.putIfAbsent()</li> \
@@ -217,9 +217,9 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                             <li> IgniteCache.getAndPutIfAbsent()</li>\
                         </ul>')
                 .settings-row
-                    +checkbox('Read-through', model + '.readThrough', '"readThrough"', 'Flag indicating whether read-through caching should be used')
+                    +checkbox('Read-through', `${model}.readThrough`, '"readThrough"', 'Flag indicating whether read-through caching should be used')
                 .settings-row
-                    +checkbox('Write-through', model + '.writeThrough', '"writeThrough"', 'Flag indicating whether write-through caching should be used')
+                    +checkbox('Write-through', `${model}.writeThrough`, '"writeThrough"', 'Flag indicating whether write-through caching should be used')
                 .settings-row
                     +ignite-form-group
                         ignite-form-field-label
@@ -228,23 +228,23 @@ mixin hibernateField(name, model, items, valid, save, newItem)
                             | Cache write-behind settings#[br]
                             | Write-behind is a special mode when updates to cache accumulated and then asynchronously flushed to persistent store as a bulk operation
                         .group-content
-                            -var enabled = model + '.writeBehindEnabled'
+                            -var enabled = `${model}.writeBehindEnabled`
 
                             .details-row
                                 +checkbox('Enabled', enabled, '"writeBehindEnabled"', 'Flag indicating whether Ignite should use write-behind behaviour for the cache store')
                             .details-row
-                                +number('Batch size:', model + '.writeBehindBatchSize', '"writeBehindBatchSize"', enabled, '512', '1',
+                                +number('Batch size:', `${model}.writeBehindBatchSize`, '"writeBehindBatchSize"', enabled, '512', '1',
                                     'Maximum batch size for write-behind cache store operations<br/>\
                                      Store operations(get or remove) are combined in a batch of this size to be passed to cache store')
                             .details-row
-                                +number('Flush size:', model + '.writeBehindFlushSize', '"writeBehindFlushSize"', enabled, '10240', '0',
+                                +number('Flush size:', `${model}.writeBehindFlushSize`, '"writeBehindFlushSize"', enabled, '10240', '0',
                                     'Maximum size of the write-behind cache<br/>\
                                      If cache size exceeds this value, all cached items are flushed to the cache store and write cache is cleared')
                             .details-row
-                                +number('Flush frequency:', model + '.writeBehindFlushFrequency', '"writeBehindFlushFrequency"', enabled, '5000', '0',
+                                +number('Flush frequency:', `${model}.writeBehindFlushFrequency`, '"writeBehindFlushFrequency"', enabled, '5000', '0',
                                     'Frequency with which write-behind cache is flushed to the cache store in milliseconds')
                             .details-row
-                                +number('Flush threads count:', model + '.writeBehindFlushThreadCount', '"writeBehindFlushThreadCount"', enabled, '1', '1',
+                                +number('Flush threads count:', `${model}.writeBehindFlushThreadCount`, '"writeBehindFlushThreadCount"', enabled, '1', '1',
                                     'Number of threads that will perform cache flushing')
             .col-sm-6
                 +preview-xml-java(model, 'cacheStore', 'domains')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.pug
similarity index 83%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.pug
index 412ca3ad538b8..61532ab3f54c6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'atomics'
 -var model = 'backupItem.atomicConfiguration'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Atomic configuration
         ignite-form-field-tooltip.tipLabel
@@ -28,11 +28,11 @@ include /app/helpers/jade/mixins.jade
             | Atomics are distributed across the cluster, essentially enabling performing atomic operations (such as increment-and-get or compare-and-set) with the same globally-visible value#[br]
             | #[a(href="https://apacheignite.readme.io/docs/atomic-types" target="_blank") More info]
         ignite-form-revert 
-    .panel-collapse(role='tabpanel' bs-collapse-target='' id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target='' id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Cache mode:', model + '.cacheMode', '"cacheMode"', 'true', 'PARTITIONED',
+                    +dropdown('Cache mode:', `${model}.cacheMode`, '"cacheMode"', 'true', 'PARTITIONED',
                         '[\
                             {value: "LOCAL", label: "LOCAL"},\
                             {value: "REPLICATED", label: "REPLICATED"},\
@@ -45,10 +45,10 @@ include /app/helpers/jade/mixins.jade
                             <li>Local - in this mode caches residing on different grid nodes will not know about each other</li>\
                         </ul>')
                 .settings-row
-                    +number('Sequence reserve:', model + '.atomicSequenceReserveSize', '"atomicSequenceReserveSize"', 'true', '1000', '0',
+                    +number('Sequence reserve:', `${model}.atomicSequenceReserveSize`, '"atomicSequenceReserveSize"', 'true', '1000', '0',
                         'Default number of sequence values reserved for IgniteAtomicSequence instances<br/>\
                         After a certain number has been reserved, consequent increments of sequence will happen locally, without communication with other nodes, until the next reservation has to be made')
-                .settings-row(ng-show='!(#{model}.cacheMode && #{model}.cacheMode != "PARTITIONED")')
+                .settings-row(ng-show=`!(${model}.cacheMode && ${model}.cacheMode != "PARTITIONED")`)
                     +number('Backups:', model + '.backups', '"backups"', 'true', '0', '0', 'Number of backup nodes')
             .col-sm-6
                 +preview-xml-java(model, 'clusterAtomics')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.pug
similarity index 80%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.pug
index 0366ed87deba4..cac122b6b00ef 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.pug
@@ -14,36 +14,36 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'attributes'
 -var model = 'backupItem'
 -var userAttributes = model + '.attributes'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label User attributes
         ignite-form-field-tooltip.tipLabel
             | Configuration for Ignite user attributes
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +ignite-form-group(ng-model='#{userAttributes}' ng-form='#{form}')
+                    +ignite-form-group(ng-model=`${userAttributes}` ng-form=form)
                         ignite-form-field-label
                             | User attributes
                         ignite-form-group-tooltip
                             | User-defined attributes to add to node
                         ignite-form-group-add(ng-click='tableNewItem(attributesTbl)')
                             | Add user attribute
-                        .group-content-empty(ng-if='!((#{userAttributes} && #{userAttributes}.length > 0) || tableNewItemActive(attributesTbl))')
+                        .group-content-empty(ng-if=`!((${userAttributes} && ${userAttributes}.length > 0) || tableNewItemActive(attributesTbl))`)
                             | Not defined
-                        .group-content(ng-show='(#{userAttributes} && #{userAttributes}.length > 0) || tableNewItemActive(attributesTbl)')
+                        .group-content(ng-show=`(${userAttributes} && ${userAttributes}.length > 0) || tableNewItemActive(attributesTbl)`)
                             table.links-edit(id='attributes' st-table=userAttributes)
                                 tbody
-                                    tr(ng-repeat='item in #{userAttributes} track by $index')
+                                    tr(ng-repeat=`item in ${userAttributes} track by $index`)
                                         td.col-sm-12(ng-hide='tableEditing(attributesTbl, $index)')
                                             a.labelFormField(ng-click='tableStartEdit(backupItem, attributesTbl, $index)') {{item.name}} = {{item.value}}
                                             +btn-remove('tableRemove(backupItem, attributesTbl, $index)', '"Remove attribute"')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
similarity index 92%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/binary.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
index f8ec8f9375b76..ee9f37bd1645b 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
@@ -14,22 +14,22 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'binary'
 -var model = 'backupItem.binaryConfiguration'
 -var types = model + '.typeConfigurations'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Binary configuration
         ignite-form-field-tooltip.tipLabel
             | Configuration of specific binary types#[br]
             | #[a(href="https://apacheignite.readme.io/docs/binary-marshaller" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +java-class('ID mapper:', model + '.idMapper', '"idMapper"', 'true', 'false',
@@ -45,11 +45,11 @@ include /app/helpers/jade/mixins.jade
                             | Type configurations
                         ignite-form-group-tooltip
                             | Configuration properties for binary types
-                        ignite-form-group-add(ng-click='#{types}.push({})')
+                        ignite-form-group-add(ng-click=`${types}.push({})`)
                             | Add new type configuration.
-                        .group-content-empty(ng-if='!#{types}.length')
+                        .group-content-empty(ng-if=`!${types}.length`)
                             | Not defined
-                        .group-content(ng-repeat='model in #{types} track by $index')
+                        .group-content(ng-repeat=`model in ${types} track by $index`)
                             hr(ng-if='$index !== 0')
                             .settings-row
                                 +java-class-autofocus('Type name:', 'model.typeName', '"typeName" + $index', 'true', 'true', 'true', 'Type name')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.pug
similarity index 83%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.pug
index 41a72c1c5a1b2..c2926f57f2836 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.pug
@@ -14,20 +14,20 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'cacheKeyCfg'
 -var model = 'backupItem.cacheKeyConfiguration'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Cache key configuration
         ignite-form-field-tooltip.tipLabel
             | Cache key configuration allows to collocate objects in a partitioned cache based on field in cache key without explicit usage of annotations on user classes
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +ignite-form-group()
@@ -35,11 +35,11 @@ include /app/helpers/jade/mixins.jade
                             | Cache key configuration
                         ignite-form-group-tooltip
                             | Cache key configuration
-                        ignite-form-group-add(ng-click='#{model}.push({})')
+                        ignite-form-group-add(ng-click=`${model}.push({})`)
                             | Add new cache key configuration
-                        .group-content-empty(ng-if='!#{model}.length')
+                        .group-content-empty(ng-if=`!${model}.length`)
                             | Not defined
-                        .group-content(ng-repeat='model in #{model} track by $index')
+                        .group-content(ng-repeat=`model in ${model} track by $index`)
                             hr(ng-if='$index !== 0')
                             .settings-row
                                 +java-class-autofocus('Type name:', 'model.typeName', '"cacheKeyTypeName" + $index', 'true', 'true', 'true', 'Type name')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.pug
similarity index 86%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.pug
index 259909eac9ed9..716e082c68f48 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'checkpoint'
 -var model = 'backupItem.checkpointSpi'
@@ -22,15 +22,15 @@ include /app/helpers/jade/mixins.jade
 -var CacheCheckpoint = 'model.kind === "Cache"'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Checkpointing
         ignite-form-field-tooltip.tipLabel
             | Checkpointing provides an ability to save an intermediate job state#[br]
             | #[a(href="http://apacheignite.readme.io/docs/checkpointing" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row(ng-init='checkpointSpiTbl={type: "checkpointSpi", model: "checkpointSpi", focusId: "kind", ui: "checkpoint-table"}')
                     +ignite-form-group()
@@ -40,9 +40,9 @@ include /app/helpers/jade/mixins.jade
                             | Checkpoint SPI configurations
                         ignite-form-group-add(ng-click='tableNewItem(checkpointSpiTbl)')
                             | Add checkpoint SPI
-                        .group-content-empty(ng-if='!(#{model} && #{model}.length > 0)')
+                        .group-content-empty(ng-if=`!(${model} && ${model}.length > 0)`)
                             | Not defined
-                        .group-content(ng-show='#{model} && #{model}.length > 0' ng-repeat='model in #{model} track by $index')
+                        .group-content(ng-show=`${model} && ${model}.length > 0` ng-repeat=`model in ${model} track by $index`)
                             hr(ng-if='$index != 0')
                             .settings-row
                                 +dropdown-required-autofocus('Checkpoint SPI:', 'model.kind', '"checkpointKind" + $index', 'true', 'true', 'Choose checkpoint configuration variant', '[\
@@ -63,21 +63,21 @@ include /app/helpers/jade/mixins.jade
                                     +table-remove-button(model, 'Remove Checkpoint SPI')
 
                             div(ng-show='model.kind === "FS"')
-                                include ./checkpoint/fs.jade
+                                include ./checkpoint/fs
 
                             div(ng-show=CacheCheckpoint)
                                 .settings-row
-                                    +dropdown-required-empty('Cache:', 'model.Cache.cache', '"checkpointCacheCache"+ $index', 'true', CacheCheckpoint,
+                                    +dropdown-required-empty('Cache:', 'model.Cache.cache', '"checkpointCacheCache" + $index', 'true', CacheCheckpoint,
                                         'Choose cache', 'No caches configured for current cluster', 'clusterCaches', 'Cache to use for storing checkpoints')
                                 .settings-row
                                     +java-class('Listener:', 'model.Cache.checkpointListener', '"checkpointCacheListener" + $index', 'true', 'false',
                                         'Checkpoint listener implementation class name', CacheCheckpoint)
 
                             div(ng-show='model.kind === "S3"')
-                                include ./checkpoint/s3.jade
+                                include ./checkpoint/s3
 
                             div(ng-show='model.kind === "JDBC"')
-                                include ./checkpoint/jdbc.jade
+                                include ./checkpoint/jdbc
 
                             .settings-row(ng-show=CustomCheckpoint)
                                 +java-class('Class name:', 'model.Custom.className', '"checkpointCustomClassName" + $index', 'true', CustomCheckpoint,
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.pug
similarity index 85%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.pug
index 6ec4535dbdcf5..04cc7fb27f48a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'checkpointFsPaths'
 -var dirPaths = 'model.FS.directoryPaths'
@@ -30,19 +30,19 @@ include /app/helpers/jade/mixins.jade
         ignite-form-group-add(ng-click='(group.add = [{}])')
             | Add new path
 
-        .group-content(ng-if='#{dirPaths}.length')
+        .group-content(ng-if=`${dirPaths}.length`)
             -var model = 'obj.model';
             -var name = '"edit" + $index'
-            -var valid = form + '[' + name + '].$valid'
+            -var valid = `${form}[${name}].$valid`
             -var save = dirPaths + '[$index] = ' + model
 
-            div(ng-repeat='item in #{dirPaths} track by $index' ng-init='obj = {}')
+            div(ng-repeat=`item in ${dirPaths} track by $index` ng-init='obj = {}')
                 label.col-xs-12.col-sm-12.col-md-12
                     .indexField
                         | {{ $index+1 }})
                     +table-remove-conditional-button(dirPaths, 'true', 'Remove path', 'item')
                     span(ng-hide='field.edit')
-                        a.labelFormField(ng-click='(field.edit = true) && (#{model} = item)') {{ item }}
+                        a.labelFormField(ng-click=`(field.edit = true) && (${model} = item)`) {{ item }}
                     span(ng-if='field.edit')
                         +table-text-field(name, model, dirPaths, valid, save, 'Input directory path', false)
                             +table-save-button(valid, save, false)
@@ -50,7 +50,7 @@ include /app/helpers/jade/mixins.jade
         .group-content(ng-repeat='field in group.add')
             -var model = 'new';
             -var name = '"new"'
-            -var valid = form + '[' + name + '].$valid'
+            -var valid = `${form}[${name}].$valid`
             -var save = dirPaths + '.push(' + model + ')'
 
             div
@@ -58,7 +58,7 @@ include /app/helpers/jade/mixins.jade
                     +table-text-field(name, model, dirPaths, valid, save, 'Input directory path', true)
                         +table-save-button(valid, save, true)
                         +unique-feedback(name, uniqueTip)
-        .group-content-empty(ng-if='!(#{dirPaths}.length) && !group.add.length')
+        .group-content-empty(ng-if=`!(${dirPaths}.length) && !group.add.length`)
             | Not defined
 
 .settings-row
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.pug
similarity index 98%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.pug
index 5a13337f564f1..ea67977e3629a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var jdbcCheckpoint = 'model.kind === "JDBC"'
 
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.pug
similarity index 99%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.pug
index 6531897130051..16be6c0a818a6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var credentialsModel = 'model.S3.awsCredentials'
 -var clientCfgModel = 'model.S3.clientConfiguration'
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision.pug
similarity index 75%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/collision.pug
index 91676df7cc1be..2f58e0a6d2aab 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision.pug
@@ -14,22 +14,22 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'collision'
 -var model = 'backupItem.collision'
 -var modelCollisionKind = model + '.kind';
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Collision configuration
         ignite-form-field-tooltip.tipLabel
             | Configuration Collision SPI allows to regulate how grid jobs get executed when they arrive on a destination node for execution#[br]
             | #[a(href="https://apacheignite.readme.io/docs/job-scheduling" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +dropdown('CollisionSpi:', modelCollisionKind, '"collisionKind"', 'true', '',
@@ -48,16 +48,16 @@ include /app/helpers/jade/mixins.jade
                             <li>Custom - custom CollisionSpi implementation</li>\
                             <li>Default - jobs are activated immediately on arrival to mapped node</li>\
                         </ul>')
-                .settings-row(ng-show='#{modelCollisionKind} !== "Noop"')
+                .settings-row(ng-show=`${modelCollisionKind} !== 'Noop'`)
                     .panel-details
-                        div(ng-show='#{modelCollisionKind} === "JobStealing"')
-                            include ./collision/job-stealing.jade
-                        div(ng-show='#{modelCollisionKind} === "FifoQueue"')
-                            include ./collision/fifo-queue.jade
-                        div(ng-show='#{modelCollisionKind} === "PriorityQueue"')
-                            include ./collision/priority-queue.jade
-                        div(ng-show='#{modelCollisionKind} === "Custom"')
-                            include ./collision/custom.jade
+                        div(ng-show=`${modelCollisionKind} === 'JobStealing'`)
+                            include ./collision/job-stealing
+                        div(ng-show=`${modelCollisionKind} === 'FifoQueue'`)
+                            include ./collision/fifo-queue
+                        div(ng-show=`${modelCollisionKind} === 'PriorityQueue'`)
+                            include ./collision/priority-queue
+                        div(ng-show=`${modelCollisionKind} === 'Custom'`)
+                            include ./collision/custom
             .col-sm-6
                 -var model = 'backupItem.collision'
                 +preview-xml-java(model, 'clusterCollision')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.pug
index 8e77ac42922c6..dc5dee09f7f6f 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.pug
@@ -14,11 +14,11 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.collision.Custom'
 -var required = 'backupItem.collision.kind === "Custom"'
 
 div
     .details-row
-        +java-class('Class:', model + '.class', '"collisionCustom"', 'true', required, 'CollisionSpi implementation class', required)
+        +java-class('Class:', `${model}.class`, '"collisionCustom"', 'true', required, 'CollisionSpi implementation class', required)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.pug
similarity index 78%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.pug
index cd8b6a3c92e5b..159b463ab9a89 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.collision.FifoQueue'
 
 div
     .details-row
-        +number('Parallel jobs number:', model + '.parallelJobsNumber', '"fifoParallelJobsNumber"', 'true', 'availableProcessors * 2', '1',
+        +number('Parallel jobs number:', `${model}.parallelJobsNumber`, '"fifoParallelJobsNumber"', 'true', 'availableProcessors * 2', '1',
             'Number of jobs that can be executed in parallel')
     .details-row
-        +number('Wait jobs number:', model + '.waitingJobsNumber', '"fifoWaitingJobsNumber"', 'true', 'Integer.MAX_VALUE', '0',
+        +number('Wait jobs number:', `${model}.waitingJobsNumber`, '"fifoWaitingJobsNumber"', 'true', 'Integer.MAX_VALUE', '0',
             'Maximum number of jobs that are allowed to wait in waiting queue')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.pug
similarity index 73%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.pug
index dbe0478344492..d10a02eb56e23 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.pug
@@ -14,29 +14,29 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.collision.JobStealing'
--var stealingAttributes = model + '.stealingAttributes'
+-var stealingAttributes = `${model}.stealingAttributes`
 
 div
     .details-row
-        +number('Active jobs threshold:', model + '.activeJobsThreshold', '"jsActiveJobsThreshold"', 'true', '95', '0',
+        +number('Active jobs threshold:', `${model}.activeJobsThreshold`, '"jsActiveJobsThreshold"', 'true', '95', '0',
             'Number of jobs that can be executed in parallel')
     .details-row
-        +number('Wait jobs threshold:', model + '.waitJobsThreshold', '"jsWaitJobsThreshold"', 'true', '0', '0',
+        +number('Wait jobs threshold:', `${model}.waitJobsThreshold`, '"jsWaitJobsThreshold"', 'true', '0', '0',
             'Job count threshold at which this node will start stealing jobs from other nodes')
     .details-row
-        +number('Message expire time:', model + '.messageExpireTime', '"jsMessageExpireTime"', 'true', '1000', '1',
+        +number('Message expire time:', `${model}.messageExpireTime`, '"jsMessageExpireTime"', 'true', '1000', '1',
             'Message expire time in ms')
     .details-row
-        +number('Maximum stealing attempts:', model + '.maximumStealingAttempts', '"jsMaximumStealingAttempts"', 'true', '5', '1',
+        +number('Maximum stealing attempts:', `${model}.maximumStealingAttempts`, '"jsMaximumStealingAttempts"', 'true', '5', '1',
             'Maximum number of attempts to steal job by another node')
     .details-row
-        +checkbox('Stealing enabled', model + '.stealingEnabled', '"jsStealingEnabled"',
+        +checkbox('Stealing enabled', `${model}.stealingEnabled`, '"jsStealingEnabled"',
             'Node should attempt to steal jobs from other nodes')
     .details-row
-        +java-class('External listener:', model + '.externalCollisionListener', '"jsExternalCollisionListener"', 'true', 'false',
+        +java-class('External listener:', `${model}.externalCollisionListener`, '"jsExternalCollisionListener"', 'true', 'false',
             'Listener to be set for notification of external collision events', 'backupItem.collision.kind === "JobStealing"')
     .details-row
         +ignite-form-group
@@ -46,12 +46,12 @@ div
                 | Configuration parameter to enable stealing to/from only nodes that have these attributes set
             ignite-form-group-add(ng-click='tableNewItem(stealingAttributesTbl)')
                 | Add stealing attribute
-            .group-content-empty(ng-if='!((#{stealingAttributes} && #{stealingAttributes}.length > 0) || tableNewItemActive(stealingAttributesTbl))')
+            .group-content-empty(ng-if=`!((${stealingAttributes} && ${stealingAttributes}.length > 0) || tableNewItemActive(stealingAttributesTbl))`)
                 | Not defined
-            .group-content(ng-show='(#{stealingAttributes} && #{stealingAttributes}.length > 0) || tableNewItemActive(stealingAttributesTbl)')
+            .group-content(ng-show=`(${stealingAttributes} && ${stealingAttributes}.length > 0) || tableNewItemActive(stealingAttributesTbl)`)
                 table.links-edit(id='attributes' st-table=stealingAttributes)
                     tbody
-                        tr(ng-repeat='item in #{stealingAttributes} track by $index')
+                        tr(ng-repeat=`item in ${stealingAttributes} track by $index`)
                             td.col-sm-12(ng-show='!tableEditing(stealingAttributesTbl, $index)')
                                 a.labelFormField(ng-click='tableStartEdit(backupItem, stealingAttributesTbl, $index)') {{item.name}} = {{item.value}}
                                 +btn-remove('tableRemove(backupItem, stealingAttributesTbl, $index)', '"Remove attribute"')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.pug
similarity index 64%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.pug
index 6f52ee0f07f19..04056df2d9c77 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.pug
@@ -14,29 +14,29 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.collision.PriorityQueue'
 
 div
     .details-row
-        +number('Parallel jobs number:', model + '.parallelJobsNumber', '"priorityParallelJobsNumber"', 'true', 'availableProcessors * 2', '1',
+        +number('Parallel jobs number:', `${model}.parallelJobsNumber`, '"priorityParallelJobsNumber"', 'true', 'availableProcessors * 2', '1',
             'Number of jobs that can be executed in parallel')
     .details-row
-        +number('Waiting jobs number:', model + '.waitingJobsNumber', '"priorityWaitingJobsNumber"', 'true', 'Integer.MAX_VALUE', '0',
+        +number('Waiting jobs number:', `${model}.waitingJobsNumber`, '"priorityWaitingJobsNumber"', 'true', 'Integer.MAX_VALUE', '0',
             'Maximum number of jobs that are allowed to wait in waiting queue')
     .details-row
-        +text('Priority attribute key:', model + '.priorityAttributeKey', '"priorityPriorityAttributeKey"', 'false', 'grid.task.priority',
+        +text('Priority attribute key:', `${model}.priorityAttributeKey`, '"priorityPriorityAttributeKey"', 'false', 'grid.task.priority',
             'Task priority attribute key')
     .details-row
-        +text('Job priority attribute key:', model + '.jobPriorityAttributeKey', '"priorityJobPriorityAttributeKey"', 'false', 'grid.job.priority',
+        +text('Job priority attribute key:', `${model}.jobPriorityAttributeKey`, '"priorityJobPriorityAttributeKey"', 'false', 'grid.job.priority',
             'Job priority attribute key')
     .details-row
-        +number('Default priority:', model + '.defaultPriority', '"priorityDefaultPriority"', 'true', '0', '0',
+        +number('Default priority:', `${model}.defaultPriority`, '"priorityDefaultPriority"', 'true', '0', '0',
             'Default priority to use if a job does not have priority attribute set')
     .details-row
-        +number('Starvation increment:', model + '.starvationIncrement', '"priorityStarvationIncrement"', 'true', '1', '0',
+        +number('Starvation increment:', `${model}.starvationIncrement`, '"priorityStarvationIncrement"', 'true', '1', '0',
             'Value to increment job priority by every time a lower priority job gets behind a higher priority job')
     .details-row
-        +checkbox('Starvation prevention enabled', model + '.starvationPreventionEnabled', '"priorityStarvationPreventionEnabled"',
+        +checkbox('Starvation prevention enabled', `${model}.starvationPreventionEnabled`, '"priorityStarvationPreventionEnabled"',
             'Job starvation prevention is enabled')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
similarity index 56%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/communication.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
index 047c9a28f5309..cd93565720e2b 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'communication'
 -var model = 'backupItem'
 -var communication = model + '.communication'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Communication
         ignite-form-field-tooltip.tipLabel
@@ -29,53 +29,53 @@ include /app/helpers/jade/mixins.jade
             | Provide basic plumbing to send and receive grid messages and is utilized for all distributed grid operations#[br]
             | #[a(href="https://apacheignite.readme.io/docs/network-config" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Timeout:', model + '.networkTimeout', '"commNetworkTimeout"', 'true', '5000', '1', 'Maximum timeout in milliseconds for network requests')
+                    +number('Timeout:', `${model}.networkTimeout`, '"commNetworkTimeout"', 'true', '5000', '1', 'Maximum timeout in milliseconds for network requests')
                 .settings-row
-                    +number('Send retry delay:', model + '.networkSendRetryDelay', '"networkSendRetryDelay"', 'true', '1000', '1', 'Interval in milliseconds between message send retries')
+                    +number('Send retry delay:', `${model}.networkSendRetryDelay`, '"networkSendRetryDelay"', 'true', '1000', '1', 'Interval in milliseconds between message send retries')
                 .settings-row
-                    +number('Send retry count:', model + '.networkSendRetryCount', '"networkSendRetryCount"', 'true', '3', '1', 'Message send retries count')
+                    +number('Send retry count:', `${model}.networkSendRetryCount`, '"networkSendRetryCount"', 'true', '3', '1', 'Message send retries count')
                 .settings-row
-                    +number('Discovery startup delay:', model + '.discoveryStartupDelay', '"discoveryStartupDelay"', 'true', '60000', '1', 'This value is used to expire messages from waiting list whenever node discovery discrepancies happen')
+                    +number('Discovery startup delay:', `${model}.discoveryStartupDelay`, '"discoveryStartupDelay"', 'true', '60000', '1', 'This value is used to expire messages from waiting list whenever node discovery discrepancies happen')
                 .settings-row
-                    +java-class('Communication listener:', communication + '.listener', '"comListener"', 'true', 'false', 'Listener of communication events')
+                    +java-class('Communication listener:', `${communication}.listener`, '"comListener"', 'true', 'false', 'Listener of communication events')
                 .settings-row
-                    +text-ip-address('Local IP address:', communication + '.localAddress', '"comLocalAddress"', 'true', '0.0.0.0',
+                    +text-ip-address('Local IP address:', `${communication}.localAddress`, '"comLocalAddress"', 'true', '0.0.0.0',
                         'Local host address for socket binding<br/>\
                         If not specified use all available addres on local host')
                 .settings-row
-                    +number-min-max('Local port:', communication + '.localPort', '"comLocalPort"', 'true', '47100', '1024', '65535', 'Local port for socket binding')
+                    +number-min-max('Local port:', `${communication}.localPort`, '"comLocalPort"', 'true', '47100', '1024', '65535', 'Local port for socket binding')
                 .settings-row
-                    +number('Local port range:', communication + '.localPortRange', '"comLocalPortRange"', 'true', '100', '1', 'Local port range for local host ports')
+                    +number('Local port range:', `${communication}.localPortRange`, '"comLocalPortRange"', 'true', '100', '1', 'Local port range for local host ports')
                 .settings-row
-                    +number-min-max('Shared memory port:', communication + '.sharedMemoryPort', '"sharedMemoryPort"', 'true', '48100', '-1', '65535',
+                    +number-min-max('Shared memory port:', `${communication}.sharedMemoryPort`, '"sharedMemoryPort"', 'true', '48100', '-1', '65535',
                         'Local port to accept shared memory connections<br/>\
                         If set to #[b -1] shared memory communication will be disabled')
                 .settings-row
-                    +number('Idle connection timeout:', communication + '.idleConnectionTimeout', '"idleConnectionTimeout"', 'true', '30000', '1',
+                    +number('Idle connection timeout:', `${communication}.idleConnectionTimeout`, '"idleConnectionTimeout"', 'true', '30000', '1',
                         'Maximum idle connection timeout upon which a connection to client will be closed')
                 .settings-row
-                    +number('Connect timeout:', communication + '.connectTimeout', '"connectTimeout"', 'true', '5000', '0', 'Connect timeout used when establishing connection with remote nodes')
+                    +number('Connect timeout:', `${communication}.connectTimeout`, '"connectTimeout"', 'true', '5000', '0', 'Connect timeout used when establishing connection with remote nodes')
                 .settings-row
-                    +number('Maximum connect timeout:', communication + '.maxConnectTimeout', '"maxConnectTimeout"', 'true', '600000', '0', 'Maximum connect timeout')
+                    +number('Maximum connect timeout:', `${communication}.maxConnectTimeout`, '"maxConnectTimeout"', 'true', '600000', '0', 'Maximum connect timeout')
                 .settings-row
-                    +number('Reconnect count:', communication + '.reconnectCount', '"comReconnectCount"', 'true', '10', '1',
+                    +number('Reconnect count:', `${communication}.reconnectCount`, '"comReconnectCount"', 'true', '10', '1',
                         'Maximum number of reconnect attempts used when establishing connection with remote nodes')
                 .settings-row
-                    +number('Socket send buffer:', communication + '.socketSendBuffer', '"socketSendBuffer"', 'true', '32768', '0', 'Send buffer size for sockets created or accepted by this SPI')
+                    +number('Socket send buffer:', `${communication}.socketSendBuffer`, '"socketSendBuffer"', 'true', '32768', '0', 'Send buffer size for sockets created or accepted by this SPI')
                 .settings-row
-                    +number('Socket receive buffer:', communication + '.socketReceiveBuffer', '"socketReceiveBuffer"', 'true', '32768', '0', 'Receive buffer size for sockets created or accepted by this SPI')
+                    +number('Socket receive buffer:', `${communication}.socketReceiveBuffer`, '"socketReceiveBuffer"', 'true', '32768', '0', 'Receive buffer size for sockets created or accepted by this SPI')
                 .settings-row
-                    +number('Slow client queue limit:', communication + '.slowClientQueueLimit', '"slowClientQueueLimit"', 'true', '0', '0', 'Slow client queue limit')
+                    +number('Slow client queue limit:', `${communication}.slowClientQueueLimit`, '"slowClientQueueLimit"', 'true', '0', '0', 'Slow client queue limit')
                 .settings-row
-                    +number('Ack send threshold:', communication + '.ackSendThreshold', '"ackSendThreshold"', 'true', '16', '1', 'Number of received messages per connection to node after which acknowledgment message is sent')
+                    +number('Ack send threshold:', `${communication}.ackSendThreshold`, '"ackSendThreshold"', 'true', '16', '1', 'Number of received messages per connection to node after which acknowledgment message is sent')
                 .settings-row
-                    +number('Message queue limit:', communication + '.messageQueueLimit', '"messageQueueLimit"', 'true', '1024', '0', 'Message queue limit for incoming and outgoing messages')
+                    +number('Message queue limit:', `${communication}.messageQueueLimit`, '"messageQueueLimit"', 'true', '1024', '0', 'Message queue limit for incoming and outgoing messages')
                 .settings-row
-                    +number('Unacknowledged messages:', communication + '.unacknowledgedMessagesBufferSize', '"unacknowledgedMessagesBufferSize"', 'true', '0', '0',
+                    +number('Unacknowledged messages:', `${communication}.unacknowledgedMessagesBufferSize`, '"unacknowledgedMessagesBufferSize"', 'true', '0', '0',
                         'Maximum number of stored unacknowledged messages per connection to node<br/>\
                         If specified non zero value it should be\
                         <ul>\
@@ -83,18 +83,18 @@ include /app/helpers/jade/mixins.jade
                             <li>At least message queue limit * 5</li>\
                         </ul>')
                 .settings-row
-                    +number('Socket write timeout:', communication + '.socketWriteTimeout', '"socketWriteTimeout"', 'true', '2000', '0', 'Socket write timeout')
+                    +number('Socket write timeout:', `${communication}.socketWriteTimeout`, '"socketWriteTimeout"', 'true', '2000', '0', 'Socket write timeout')
                 .settings-row
-                    +number('Selectors count:', communication + '.selectorsCount', '"selectorsCount"', 'true', 'min(4, availableProcessors)', '1', 'Count of selectors te be used in TCP server')
+                    +number('Selectors count:', `${communication}.selectorsCount`, '"selectorsCount"', 'true', 'min(4, availableProcessors)', '1', 'Count of selectors te be used in TCP server')
                 .settings-row
-                    +java-class('Address resolver:', communication + '.addressResolver', '"comAddressResolver"', 'true', 'false', 'Provides resolution between external and internal addresses')
+                    +java-class('Address resolver:', `${communication}.addressResolver`, '"comAddressResolver"', 'true', 'false', 'Provides resolution between external and internal addresses')
                 .settings-row
-                    +checkbox('Direct buffer', communication + '.directBuffer', '"directBuffer"',
+                    +checkbox('Direct buffer', `${communication}.directBuffer`, '"directBuffer"',
                     'If value is true, then SPI will use ByteBuffer.allocateDirect(int) call<br/>\
                     Otherwise, SPI will use ByteBuffer.allocate(int) call')
                 .settings-row
-                    +checkbox('Direct send buffer', communication + '.directSendBuffer', '"directSendBuffer"', 'Flag defining whether direct send buffer should be used')
+                    +checkbox('Direct send buffer', `${communication}.directSendBuffer`, '"directSendBuffer"', 'Flag defining whether direct send buffer should be used')
                 .settings-row
-                    +checkbox('TCP_NODELAY option', communication + '.tcpNoDelay', '"tcpNoDelay"', 'Value for TCP_NODELAY socket option')
+                    +checkbox('TCP_NODELAY option', `${communication}.tcpNoDelay`, '"tcpNoDelay"', 'Value for TCP_NODELAY socket option')
             .col-sm-6
                 +preview-xml-java(model, 'clusterCommunication')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/connector.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/connector.pug
similarity index 71%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/connector.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/connector.pug
index 40d65af250fce..b145ce28cfe26 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/connector.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/connector.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'connector'
 -var model = 'backupItem.connector'
@@ -22,83 +22,83 @@ include /app/helpers/jade/mixins.jade
 -var sslEnabled = enabled + ' && ' + model + '.sslEnabled'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Connector configuration
         ignite-form-field-tooltip.tipLabel
             | Configure HTTP REST configuration to enable HTTP server features#[br]
             | #[a(href="https://apacheignite.readme.io/docs/configuration" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +checkbox('Enabled', enabled, '"restEnabled"', 'Flag indicating whether to configure connector configuration')
                 .settings-row
-                    +text-enabled('Jetty configuration path:', model + '.jettyPath', '"connectorJettyPath"', enabled, 'false', 'Input path to Jetty configuration',
+                    +text-enabled('Jetty configuration path:', `${model}.jettyPath`, '"connectorJettyPath"', enabled, 'false', 'Input path to Jetty configuration',
                         'Path, either absolute or relative to IGNITE_HOME, to Jetty XML configuration file<br/>\
                         Jetty is used to support REST over HTTP protocol for accessing Ignite APIs remotely<br/>\
                         If not provided, Jetty instance with default configuration will be started picking IgniteSystemProperties.IGNITE_JETTY_HOST and IgniteSystemProperties.IGNITE_JETTY_PORT as host and port respectively')
                 .settings-row
-                    +text-ip-address('TCP host:', model + '.host', '"connectorHost"', enabled, 'IgniteConfiguration#getLocalHost()',
+                    +text-ip-address('TCP host:', `${model}.host`, '"connectorHost"', enabled, 'IgniteConfiguration#getLocalHost()',
                         'Host for TCP binary protocol server<br/>\
                         This can be either an IP address or a domain name<br/>\
                         If not defined, system - wide local address will be used IgniteConfiguration#getLocalHost()<br/>\
                         You can also use "0.0.0.0" value to bind to all locally - available IP addresses')
                 .settings-row
-                    +number-min-max('TCP port:', model + '.port', '"connectorPort"', enabled, '11211', '1024', '65535', 'Port for TCP binary protocol server')
+                    +number-min-max('TCP port:', `${model}.port`, '"connectorPort"', enabled, '11211', '1024', '65535', 'Port for TCP binary protocol server')
                 .settings-row
-                    +number('TCP port range:', model + '.portRange', '"connectorPortRange"', enabled, '100', '1', 'Number of ports for TCP binary protocol server to try if configured port is already in use')
+                    +number('TCP port range:', `${model}.portRange`, '"connectorPortRange"', enabled, '100', '1', 'Number of ports for TCP binary protocol server to try if configured port is already in use')
                 .settings-row
-                    +number('Idle query cursor timeout:', model + '.idleQueryCursorTimeout', '"connectorIdleQueryCursorTimeout"', enabled, '600000', '0',
+                    +number('Idle query cursor timeout:', `${model}.idleQueryCursorTimeout`, '"connectorIdleQueryCursorTimeout"', enabled, '600000', '0',
                         'Reject open query cursors that is not used timeout<br/>\
                         If no fetch query request come within idle timeout, it will be removed on next check for old query cursors')
                 .settings-row
-                    +number('Idle query cursor check frequency:', model + '.idleQueryCursorCheckFrequency', '"connectorIdleQueryCursorCheckFrequency"', enabled, '60000', '0',
+                    +number('Idle query cursor check frequency:', `${model}.idleQueryCursorCheckFrequency`, '"connectorIdleQueryCursorCheckFrequency"', enabled, '60000', '0',
                         'Idle query cursors check frequency<br/>\
                         This setting is used to reject open query cursors that is not used')
                 .settings-row
-                    +number('Idle timeout:', model + '.idleTimeout', '"connectorIdleTimeout"', enabled, '7000', '0',
+                    +number('Idle timeout:', `${model}.idleTimeout`, '"connectorIdleTimeout"', enabled, '7000', '0',
                         'Idle timeout for REST server<br/>\
                         This setting is used to reject half - opened sockets<br/>\
                         If no packets come within idle timeout, the connection is closed')
                 .settings-row
-                    +number('Receive buffer size:', model + '.receiveBufferSize', '"connectorReceiveBufferSize"', enabled, '32768', '0', 'REST TCP server receive buffer size')
+                    +number('Receive buffer size:', `${model}.receiveBufferSize`, '"connectorReceiveBufferSize"', enabled, '32768', '0', 'REST TCP server receive buffer size')
                 .settings-row
-                    +number('Send buffer size:', model + '.sendBufferSize', '"connectorSendBufferSize"', enabled, '32768', '0', 'REST TCP server send buffer size')
+                    +number('Send buffer size:', `${model}.sendBufferSize`, '"connectorSendBufferSize"', enabled, '32768', '0', 'REST TCP server send buffer size')
                 .settings-row
-                    +number('Send queue limit:', model + '.sendQueueLimit', '"connectorSendQueueLimit"', enabled, 'unlimited', '0',
+                    +number('Send queue limit:', `${model}.sendQueueLimit`, '"connectorSendQueueLimit"', enabled, 'unlimited', '0',
                         'REST TCP server send queue limit<br/>\
                         If the limit exceeds, all successive writes will block until the queue has enough capacity')
                 .settings-row
-                    +checkbox-enabled('Direct buffer', model + '.directBuffer', '"connectorDirectBuffer"', enabled,
+                    +checkbox-enabled('Direct buffer', `${model}.directBuffer`, '"connectorDirectBuffer"', enabled,
                         'Flag indicating whether REST TCP server should use direct buffers<br/>\
                         A direct buffer is a buffer that is allocated and accessed using native system calls, without using JVM heap<br/>\
                         Enabling direct buffer may improve performance and avoid memory issues(long GC pauses due to huge buffer size)')
                 .settings-row
-                    +checkbox-enabled('TCP_NODELAY option', model + '.noDelay', '"connectorNoDelay"', enabled,
+                    +checkbox-enabled('TCP_NODELAY option', `${model}.noDelay`, '"connectorNoDelay"', enabled,
                         'Flag indicating whether TCP_NODELAY option should be set for accepted client connections<br/>\
                         Setting this option reduces network latency and should be enabled in majority of cases<br/>\
                         For more information, see Socket#setTcpNoDelay(boolean)')
                 .settings-row
-                    +number('Selector count:', model + '.selectorCount', '"connectorSelectorCount"', enabled, 'min(4, availableProcessors)', '1',
+                    +number('Selector count:', `${model}.selectorCount`, '"connectorSelectorCount"', enabled, 'min(4, availableProcessors)', '1',
                         'Number of selector threads in REST TCP server<br/>\
                         Higher value for this parameter may increase throughput, but also increases context switching')
                 .settings-row
-                    +number('Thread pool size:', model + '.threadPoolSize', '"connectorThreadPoolSize"', enabled, 'max(8, availableProcessors) * 2', '1',
+                    +number('Thread pool size:', `${model}.threadPoolSize`, '"connectorThreadPoolSize"', enabled, 'max(8, availableProcessors) * 2', '1',
                         'Thread pool size to use for processing of client messages (REST requests)')
                 .settings-row
-                    +java-class('Message interceptor:', model + '.messageInterceptor', '"connectorMessageInterceptor"', enabled, 'false',
+                    +java-class('Message interceptor:', `${model}.messageInterceptor`, '"connectorMessageInterceptor"', enabled, 'false',
                         'Interceptor allows to transform all objects exchanged via REST protocol<br/>\
                         For example if you use custom serialisation on client you can write interceptor to transform binary representations received from client to Java objects and later access them from java code directly')
                 .settings-row
-                    +text-enabled('Secret key:', model + '.secretKey', '"connectorSecretKey"', enabled, 'false', 'Specify to enable authentication', 'Secret key to authenticate REST requests')
+                    +text-enabled('Secret key:', `${model}.secretKey`, '"connectorSecretKey"', enabled, 'false', 'Specify to enable authentication', 'Secret key to authenticate REST requests')
                 .settings-row
-                    +checkbox-enabled('Enable SSL', model + '.sslEnabled', '"connectorSslEnabled"', enabled, 'Enables/disables SSL for REST TCP binary protocol')
+                    +checkbox-enabled('Enable SSL', `${model}.sslEnabled`, '"connectorSslEnabled"', enabled, 'Enables/disables SSL for REST TCP binary protocol')
                 .settings-row
-                    +checkbox-enabled('Enable SSL client auth', model + '.sslClientAuth', '"connectorSslClientAuth"', sslEnabled, 'Flag indicating whether or not SSL client authentication is required')
+                    +checkbox-enabled('Enable SSL client auth', `${model}.sslClientAuth`, '"connectorSslClientAuth"', sslEnabled, 'Flag indicating whether or not SSL client authentication is required')
                 .settings-row
-                    +java-class('SSL factory:', model + '.sslFactory', '"connectorSslFactory"', sslEnabled, sslEnabled,
+                    +java-class('SSL factory:', `${model}.sslFactory`, '"connectorSslFactory"', sslEnabled, sslEnabled,
                         'Instance of Factory that will be used to create an instance of SSLContext for Secure Socket Layer on TCP binary protocol')
             .col-sm-6
                 +preview-xml-java(model, 'clusterConnector')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.pug
similarity index 78%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.pug
index aa99b491c0f1c..2021c1c93f4a6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'deployment'
 -var model = 'backupItem'
@@ -28,7 +28,7 @@ include /app/helpers/jade/mixins.jade
 -var customDeployment = modelDeployment + '.kind === "Custom"'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Class deployment
         ignite-form-field-tooltip.tipLabel
@@ -36,10 +36,10 @@ include /app/helpers/jade/mixins.jade
             | #[a(href="https://apacheignite.readme.io/docs/deployment-modes" target="_blank") More info]
         ignite-form-revert
     .panel-collapse(role='tabpanel' bs-collapse-target id='deployment')
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Deployment mode:', model + '.deploymentMode', '"deploymentMode"', 'true', 'SHARED',
+                    +dropdown('Deployment mode:', `${model}.deploymentMode`, '"deploymentMode"', 'true', 'SHARED',
                         '[\
                             {value: "PRIVATE", label: "PRIVATE"},\
                             {value: "ISOLATED", label: "ISOLATED"}, \
@@ -55,13 +55,13 @@ include /app/helpers/jade/mixins.jade
                             <li>CONTINUOUS - same as SHARED deployment mode, but resources will not be undeployed even after all master nodes left grid</li>\
                         </ul>')
                 .settings-row
-                    +checkbox('Enable peer class loading', model + '.peerClassLoadingEnabled', '"peerClassLoadingEnabled"', 'Enables/disables peer class loading')
+                    +checkbox('Enable peer class loading', `${model}.peerClassLoadingEnabled`, '"peerClassLoadingEnabled"', 'Enables/disables peer class loading')
                 .settings-row
-                    +number('Missed resources cache size:', model + '.peerClassLoadingMissedResourcesCacheSize', '"peerClassLoadingMissedResourcesCacheSize"', enabled, '100', '0',
+                    +number('Missed resources cache size:', `${model}.peerClassLoadingMissedResourcesCacheSize`, '"peerClassLoadingMissedResourcesCacheSize"', enabled, '100', '0',
                         'If size greater than 0, missed resources will be cached and next resource request ignored<br/>\
                         If size is 0, then request for the resource will be sent to the remote node every time this resource is requested')
                 .settings-row
-                    +number('Pool size:', model + '.peerClassLoadingThreadPoolSize', '"peerClassLoadingThreadPoolSize"', enabled, '2', '1', 'Thread pool size to use for peer class loading')
+                    +number('Pool size:', `${model}.peerClassLoadingThreadPoolSize`, '"peerClassLoadingThreadPoolSize"', enabled, '2', '1', 'Thread pool size to use for peer class loading')
                 .settings-row
                     +ignite-form-group
                         -var uniqueTip = 'Such package already exists'
@@ -71,30 +71,30 @@ include /app/helpers/jade/mixins.jade
                         ignite-form-group-tooltip
                             | List of packages from the system classpath that need to be peer-to-peer loaded from task originating node<br/>
                             | '*' is supported at the end of the package name which means that all sub-packages and their classes are included like in Java package import clause
-                        ignite-form-group-add(ng-show='#{enabled}' ng-click='(group.add = [{}])')
+                        ignite-form-group-add(ng-show=`${enabled}` ng-click='(group.add = [{}])')
                             | Add package name.
 
-                        .group-content(ng-if=exclude + '.length')
+                        .group-content(ng-if=`${exclude}.length`)
                             -var model = 'obj.model';
                             -var name = '"edit" + $index'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = exclude + '[$index] = ' + model
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${exclude}[$index] = ${model}`
 
                             div(ng-show=enabled)
-                                div(ng-repeat='model in #{exclude} track by $index' ng-init='obj = {}')
+                                div(ng-repeat=`model in ${exclude} track by $index` ng-init='obj = {}')
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .indexField
                                             | {{ $index+1 }})
                                         +table-remove-button(exclude, 'Remove package name')
                                         span(ng-hide='field.edit')
-                                            a.labelFormField(ng-click='(field.edit = true) && (#{model} = model)') {{ model }}
+                                            a.labelFormField(ng-click=`(field.edit = true) && (${model} = model)`) {{ model }}
                                         span(ng-if='field.edit')
                                             +table-java-package-field(name, model, exclude, valid, save, false)
                                                 +table-save-button(valid, save, false)
                                                 +unique-feedback(name, uniqueTip)
 
                             div(ng-hide=enabled)
-                                div(ng-repeat='model in #{exclude} track by $index')
+                                div(ng-repeat=`model in ${exclude} track by $index`)
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .labelFormField.labelField
                                             | {{ $index+1 }})
@@ -104,8 +104,8 @@ include /app/helpers/jade/mixins.jade
                         .group-content(ng-repeat='field in group.add')
                             -var model = 'new';
                             -var name = '"new"'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = exclude + '.push(' + model + ')'
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${exclude}.push(${model})`
 
                             div(type='internal' name='Package name')
                                 label.col-xs-12.col-sm-12.col-md-12
@@ -113,7 +113,7 @@ include /app/helpers/jade/mixins.jade
                                         +table-save-button(valid, save, true)
                                         +unique-feedback(name, uniqueTip)
 
-                        .group-content-empty(ng-if='!(#{exclude}.length) && !group.add.length')
+                        .group-content-empty(ng-if=`!(${exclude}.length) && !group.add.length`)
                             | Not defined
                 .settings-row
                     +dropdown('Deployment variant:', modelDeployment + '.kind', '"deploymentKind"', 'true', 'Default',
@@ -145,16 +145,16 @@ include /app/helpers/jade/mixins.jade
                             .group-content(ng-if=uriListModel + '.length')
                                 -var model = 'obj.model';
                                 -var name = '"edit" + $index'
-                                -var valid = form + '[' + name + '].$valid'
-                                -var save = uriListModel + '[$index] = ' + model
+                                -var valid = `${form}[${name}].$valid`
+                                -var save = `${uriListModel}[$index] = ${model}`
 
-                                div(ng-repeat='model in #{uriListModel} track by $index' ng-init='obj = {}')
+                                div(ng-repeat=`model in ${uriListModel} track by $index` ng-init='obj = {}')
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .indexField
                                             | {{ $index+1 }})
                                         +table-remove-button(uriListModel, 'Remove URI')
                                         span(ng-hide='field.edit')
-                                            a.labelFormField(ng-click='(field.edit = true) && (#{model} = model)') {{ model }}
+                                            a.labelFormField(ng-click=`(field.edit = true) && (${model} = model)`) {{ model }}
                                         span(ng-if='field.edit')
                                             +table-url-field(name, model, uriListModel, valid, save, false)
                                                 +table-save-button(valid, save, false)
@@ -163,8 +163,8 @@ include /app/helpers/jade/mixins.jade
                             .group-content(ng-repeat='field in group.add')
                                 -var model = 'new';
                                 -var name = '"new"'
-                                -var valid = form + '[' + name + '].$valid'
-                                -var save = uriListModel + '.push(' + model + ')'
+                                -var valid = `${form}[${name}].$valid`
+                                -var save = `${uriListModel}.push(${model})`
 
                                 div(type='internal' name='URI')
                                     label.col-xs-12.col-sm-12.col-md-12
@@ -172,7 +172,7 @@ include /app/helpers/jade/mixins.jade
                                             +table-save-button(valid, save, true)
                                             +unique-feedback(name, uniqueTip)
 
-                            .group-content-empty(ng-if='!(#{uriListModel}.length) && !group.add.length')
+                            .group-content-empty(ng-if=`!(${uriListModel}.length) && !group.add.length`)
                                 | Not defined
                     .details-row
                         +text('Temporary directory path:', modelDeployment + '.URI.temporaryDirectoryPath', '"DeploymentURITemporaryDirectoryPath"', 'false', 'Temporary directory path',
@@ -191,16 +191,16 @@ include /app/helpers/jade/mixins.jade
                             .group-content(ng-if=scannerModel + '.length')
                                 -var model = 'obj.model';
                                 -var name = '"edit" + $index'
-                                -var valid = form + '[' + name + '].$valid'
-                                -var save = scannerModel + '[$index] = ' + model
+                                -var valid = `${form}[${name}].$valid`
+                                -var save = `${scannerModel}[$index] = ${model}`
 
-                                div(ng-repeat='model in #{scannerModel} track by $index' ng-init='obj = {}')
+                                div(ng-repeat=`model in ${scannerModel} track by $index` ng-init='obj = {}')
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .indexField
                                             | {{ $index+1 }})
                                         +table-remove-button(scannerModel, 'Remove scanner')
                                         span(ng-hide='field.edit')
-                                            a.labelFormField(ng-click='(field.edit = true) && (#{model} = model)') {{ model }}
+                                            a.labelFormField(ng-click=`(field.edit = true) && (${model} = model)`) {{ model }}
                                         span(ng-if='field.edit')
                                             +table-java-class-field('Scanner:', name, model, scannerModel, valid, save, false)
                                                 +table-save-button(valid, save, false)
@@ -209,8 +209,8 @@ include /app/helpers/jade/mixins.jade
                             .group-content(ng-repeat='field in group.add')
                                 -var model = 'new';
                                 -var name = '"new"'
-                                -var valid = form + '[' + name + '].$valid'
-                                -var save = scannerModel + '.push(' + model + ')'
+                                -var valid = `${form}[${name}].$valid`
+                                -var save = `${scannerModel}.push(${model})`
 
                                 div(type='internal' name='Scanner')
                                     label.col-xs-12.col-sm-12.col-md-12
@@ -219,19 +219,19 @@ include /app/helpers/jade/mixins.jade
                                             +table-save-button(valid, save, true)
                                             +unique-feedback(name, uniqueTip)
 
-                            .group-content-empty(ng-if='!(#{scannerModel}.length) && !group.add.length')
+                            .group-content-empty(ng-if=`!(${scannerModel}.length) && !group.add.length`)
                                 | Not defined
                     .details-row
-                        +java-class('Listener:', modelDeployment + '.URI.listener', '"DeploymentURIListener"', 'true', 'false', 'Deployment event listener', uriDeployment)
+                        +java-class('Listener:', `${modelDeployment}.URI.listener`, '"DeploymentURIListener"', 'true', 'false', 'Deployment event listener', uriDeployment)
                     .details-row
-                        +checkbox('Check MD5', modelDeployment + '.URI.checkMd5', '"DeploymentURICheckMd5"', 'Exclude files with same md5s from deployment')
+                        +checkbox('Check MD5', `${modelDeployment}.URI.checkMd5`, '"DeploymentURICheckMd5"', 'Exclude files with same md5s from deployment')
                     .details-row
-                        +checkbox('Encode URI', modelDeployment + '.URI.encodeUri', '"DeploymentURIEncodeUri"', 'URI must be encoded before usage')
+                        +checkbox('Encode URI', `${modelDeployment}.URI.encodeUri`, '"DeploymentURIEncodeUri"', 'URI must be encoded before usage')
                 .panel-details(ng-show=localDeployment)
                     .details-row
-                        +java-class('Listener:', modelDeployment + '.Local.listener', '"DeploymentLocalListener"', 'true', 'false', 'Deployment event listener', localDeployment)
+                        +java-class('Listener:', `${modelDeployment}.Local.listener`, '"DeploymentLocalListener"', 'true', 'false', 'Deployment event listener', localDeployment)
                 .panel-details(ng-show=customDeployment)
                     .details-row
-                        +java-class('Class:', modelDeployment + '.Custom.className', '"DeploymentCustom"', 'true', customDeployment, 'DeploymentSpi implementation class', customDeployment)
+                        +java-class('Class:', `${modelDeployment}.Custom.className`, '"DeploymentCustom"', 'true', customDeployment, 'DeploymentSpi implementation class', customDeployment)
             .col-sm-6
                 +preview-xml-java(model, 'clusterDeployment')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.pug
similarity index 50%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.pug
index 3bf0e29440050..af9d875651fac 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.pug
@@ -14,75 +14,75 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'discovery'
 -var model = 'backupItem.discovery'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Discovery
         ignite-form-field-tooltip.tipLabel
             | TCP/IP discovery configuration#[br]
             | #[a(href="https://apacheignite.readme.io/docs/cluster-config" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +text-ip-address('Local address:', model + '.localAddress', '"discoLocalAddress"', 'true', '228.1.2.4',
+                    +text-ip-address('Local address:', `${model}.localAddress`, '"discoLocalAddress"', 'true', '228.1.2.4',
                         'Local host IP address that discovery SPI uses<br/>\
                         If not provided a first found non-loopback address will be used')
                 .settings-row
-                    +number-min-max('Local port:', model + '.localPort', '"discoLocalPort"', 'true', '47500', '1024', '65535', 'Local port which node uses')
+                    +number-min-max('Local port:', `${model}.localPort`, '"discoLocalPort"', 'true', '47500', '1024', '65535', 'Local port which node uses')
                 .settings-row
-                    +number('Local port range:', model + '.localPortRange', '"discoLocalPortRange"', 'true', '100', '1', 'Local port range')
+                    +number('Local port range:', `${model}.localPortRange`, '"discoLocalPortRange"', 'true', '100', '1', 'Local port range')
                 .settings-row
-                    +java-class('Address resolver:', model + '.addressResolver', '"discoAddressResolver"', 'true', 'false',
+                    +java-class('Address resolver:', `${model}.addressResolver`, '"discoAddressResolver"', 'true', 'false',
                         'Provides resolution between external and internal addresses')
                 .settings-row
-                    +number('Socket timeout:', model + '.socketTimeout', '"socketTimeout"', 'true', '5000', '0', 'Socket operations timeout')
+                    +number('Socket timeout:', `${model}.socketTimeout`, '"socketTimeout"', 'true', '5000', '0', 'Socket operations timeout')
                 .settings-row
-                    +number('Acknowledgement timeout:', model + '.ackTimeout', '"ackTimeout"', 'true', '5000', '0', 'Message acknowledgement timeout')
+                    +number('Acknowledgement timeout:', `${model}.ackTimeout`, '"ackTimeout"', 'true', '5000', '0', 'Message acknowledgement timeout')
                 .settings-row
-                    +number('Max acknowledgement timeout:', model + '.maxAckTimeout', '"maxAckTimeout"', 'true', '600000', '0', 'Maximum message acknowledgement timeout')
+                    +number('Max acknowledgement timeout:', `${model}.maxAckTimeout`, '"maxAckTimeout"', 'true', '600000', '0', 'Maximum message acknowledgement timeout')
                 .settings-row
-                    +number('Network timeout:', model + '.networkTimeout', '"discoNetworkTimeout"', 'true', '5000', '1', 'Timeout to use for network operations')
+                    +number('Network timeout:', `${model}.networkTimeout`, '"discoNetworkTimeout"', 'true', '5000', '1', 'Timeout to use for network operations')
                 .settings-row
-                    +number('Join timeout:', model + '.joinTimeout', '"joinTimeout"', 'true', '0', '0',
+                    +number('Join timeout:', `${model}.joinTimeout`, '"joinTimeout"', 'true', '0', '0',
                         'Join timeout<br/>' +
                         '0 means wait forever')
                 .settings-row
-                    +number('Thread priority:', model + '.threadPriority', '"threadPriority"', 'true', '10', '1', 'Thread priority for all threads started by SPI')
+                    +number('Thread priority:', `${model}.threadPriority`, '"threadPriority"', 'true', '10', '1', 'Thread priority for all threads started by SPI')
                 .settings-row
-                    +number('Heartbeat frequency:', model + '.heartbeatFrequency', '"heartbeatFrequency"', 'true', '2000', '1', 'Heartbeat messages issuing frequency')
+                    +number('Heartbeat frequency:', `${model}.heartbeatFrequency`, '"heartbeatFrequency"', 'true', '2000', '1', 'Heartbeat messages issuing frequency')
                 .settings-row
-                    +number('Max heartbeats miss w/o init:', model + '.maxMissedHeartbeats', '"maxMissedHeartbeats"', 'true', '1', '1',
+                    +number('Max heartbeats miss w/o init:', `${model}.maxMissedHeartbeats`, '"maxMissedHeartbeats"', 'true', '1', '1',
                         'Max heartbeats count node can miss without initiating status check')
                 .settings-row
-                    +number('Max missed client heartbeats:', model + '.maxMissedClientHeartbeats', '"maxMissedClientHeartbeats"', 'true', '5', '1',
+                    +number('Max missed client heartbeats:', `${model}.maxMissedClientHeartbeats`, '"maxMissedClientHeartbeats"', 'true', '5', '1',
                         'Max heartbeats count node can miss without failing client node')
                 .settings-row
-                    +number('Topology history:', model + '.topHistorySize', '"topHistorySize"', 'true', '1000', '0', 'Size of topology snapshots history')
+                    +number('Topology history:', `${model}.topHistorySize`, '"topHistorySize"', 'true', '1000', '0', 'Size of topology snapshots history')
                 .settings-row
-                    +java-class('Discovery listener:', model + '.listener', '"discoListener"', 'true', 'false', 'Listener for grid node discovery events')
+                    +java-class('Discovery listener:', `${model}.listener`, '"discoListener"', 'true', 'false', 'Listener for grid node discovery events')
                 .settings-row
-                    +java-class('Data exchange:', model + '.dataExchange', '"dataExchange"', 'true', 'false', 'Class name of handler for initial data exchange between Ignite nodes')
+                    +java-class('Data exchange:', `${model}.dataExchange`, '"dataExchange"', 'true', 'false', 'Class name of handler for initial data exchange between Ignite nodes')
                 .settings-row
-                    +java-class('Metrics provider:', model + '.metricsProvider', '"metricsProvider"', 'true', 'false', 'Class name of metric provider to discovery SPI')
+                    +java-class('Metrics provider:', `${model}.metricsProvider`, '"metricsProvider"', 'true', 'false', 'Class name of metric provider to discovery SPI')
                 .settings-row
-                    +number('Reconnect count:', model + '.reconnectCount', '"discoReconnectCount"', 'true', '10', '1', 'Reconnect attempts count')
+                    +number('Reconnect count:', `${model}.reconnectCount`, '"discoReconnectCount"', 'true', '10', '1', 'Reconnect attempts count')
                 .settings-row
-                    +number('Statistics frequency:', model + '.statisticsPrintFrequency', '"statisticsPrintFrequency"', 'true', '0', '1', 'Statistics print frequency')
+                    +number('Statistics frequency:', `${model}.statisticsPrintFrequency`, '"statisticsPrintFrequency"', 'true', '0', '1', 'Statistics print frequency')
                 .settings-row
-                    +number('IP finder clean frequency:', model + '.ipFinderCleanFrequency', '"ipFinderCleanFrequency"', 'true', '60000', '1', 'IP finder clean frequency')
+                    +number('IP finder clean frequency:', `${model}.ipFinderCleanFrequency`, '"ipFinderCleanFrequency"', 'true', '60000', '1', 'IP finder clean frequency')
                 .settings-row
-                    +java-class('Node authenticator:', model + '.authenticator', '"authenticator"', 'true', 'false', 'Class name of node authenticator implementation')
+                    +java-class('Node authenticator:', `${model}.authenticator`, '"authenticator"', 'true', 'false', 'Class name of node authenticator implementation')
                 .settings-row
-                    +checkbox('Force server mode', model + '.forceServerMode', '"forceServerMode"', 'Force start TCP/IP discovery in server mode')
+                    +checkbox('Force server mode', `${model}.forceServerMode`, '"forceServerMode"', 'Force start TCP/IP discovery in server mode')
                 .settings-row
-                    +checkbox('Client reconnect disabled', model + '.clientReconnectDisabled', '"clientReconnectDisabled"',
+                    +checkbox('Client reconnect disabled', `${model}.clientReconnectDisabled`, '"clientReconnectDisabled"',
                         'Disable try of client to reconnect after server detected client node failure')
             .col-sm-6
                 +preview-xml-java(model, 'clusterDiscovery')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/events.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/events.pug
similarity index 78%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/events.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/events.pug
index 643ea9703d230..70fb71494a0db 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/events.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/events.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'events'
 -var model = 'backupItem'
@@ -24,18 +24,18 @@ include /app/helpers/jade/mixins.jade
 -var eventStorageCustom = modelEventStorageKind + ' === "Custom"'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Events
         ignite-form-field-tooltip.tipLabel
             | Grid events are used for notification about what happens within the grid#[br]
             | #[a(href="https://apacheignite.readme.io/docs/events" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown-multiple('Include type:', model + '.includeEventTypes', '"includeEventTypes"', true, 'Choose recorded event types', '', 'eventGroups',
+                    +dropdown-multiple('Include type:', `${model}.includeEventTypes`, '"includeEventTypes"', true, 'Choose recorded event types', '', 'eventGroups',
                     'Array of event types, which will be recorded by GridEventStorageManager#record(Event)<br/>\
                     Note, that either the include event types or the exclude event types can be established')
 
@@ -53,16 +53,16 @@ include /app/helpers/jade/mixins.jade
 
                 div(ng-show=eventStorageMemory)
                     .settings-row
-                        +number('Events expiration time:', modelEventStorage + '.Memory.expireAgeMs', '"EventStorageExpireAgeMs"', 'true', 'Long.MAX_VALUE', '1', 'All events that exceed this value will be removed from the queue when next event comes')
+                        +number('Events expiration time:', `${modelEventStorage}.Memory.expireAgeMs`, '"EventStorageExpireAgeMs"', 'true', 'Long.MAX_VALUE', '1', 'All events that exceed this value will be removed from the queue when next event comes')
                     .settings-row
-                        +number('Events queue size:', modelEventStorage + '.Memory.expireCount', '"EventStorageExpireCount"', 'true', '10000', '1', 'Events will be filtered out when new request comes')
+                        +number('Events queue size:', `${modelEventStorage}.Memory.expireCount`, '"EventStorageExpireCount"', 'true', '10000', '1', 'Events will be filtered out when new request comes')
                     .settings-row
-                        +java-class('Filter:', modelEventStorage + '.Memory.filter', '"EventStorageFilter"', 'true', 'false',
+                        +java-class('Filter:', `${modelEventStorage}.Memory.filter`, '"EventStorageFilter"', 'true', 'false',
                         'Filter for events to be recorded<br/>\
                         Should be implementation of o.a.i.lang.IgnitePredicate&lt;o.a.i.events.Event&gt;', eventStorageMemory)
 
                 .settings-row(ng-show=eventStorageCustom)
-                    +java-class('Class:', modelEventStorage + '.Custom.className', '"EventStorageCustom"', 'true', eventStorageCustom, 'Event storage implementation class name', eventStorageCustom)
+                    +java-class('Class:', `${modelEventStorage}.Custom.className`, '"EventStorageCustom"', 'true', eventStorageCustom, 'Event storage implementation class name', eventStorageCustom)
 
             .col-sm-6
                 +preview-xml-java(model, 'clusterEvents')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/failover.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/failover.pug
index 16656598c2f99..e0c2357fedb65 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem'
 -var form = 'failoverSpi'
@@ -22,15 +22,15 @@ include /app/helpers/jade/mixins.jade
 -var failoverCustom = 'model.kind === "Custom"'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Failover configuration
         ignite-form-field-tooltip.tipLabel
             | Failover SPI provides ability to supply custom logic for handling failed execution of a grid job#[br]
             | #[a(href="https://apacheignite.readme.io/docs/fault-tolerance" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row(ng-init='failoverSpiTbl={type: "failoverSpi", model: "failoverSpi", focusId: "kind", ui: "failover-table"}')
                     +ignite-form-group()
@@ -40,9 +40,9 @@ include /app/helpers/jade/mixins.jade
                             | Failover SPI configurations
                         ignite-form-group-add(ng-click='tableNewItem(failoverSpiTbl)')
                             | Add failover SPI
-                        .group-content-empty(ng-if='!(#{failoverSpi} && #{failoverSpi}.length > 0)')
+                        .group-content-empty(ng-if=`!(${failoverSpi} && ${failoverSpi}.length > 0)`)
                             | Not defined
-                        .group-content(ng-show='#{failoverSpi} && #{failoverSpi}.length > 0' ng-repeat='model in #{failoverSpi} track by $index')
+                        .group-content(ng-show=`${failoverSpi} && ${failoverSpi}.length > 0` ng-repeat=`model in ${failoverSpi} track by $index`)
                             hr(ng-if='$index != 0')
                             .settings-row
                                 +dropdown-required-autofocus('Failover SPI:', 'model.kind', '"failoverKind" + $index', 'true', 'true', 'Choose Failover SPI', '[\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
similarity index 71%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
index b2ce71fa14e53..be56a6c44d08f 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var model = 'backupItem'
@@ -28,20 +28,20 @@ include /app/helpers/jade/mixins.jade
             | Common cluster configuration#[br]
             | #[a(href="https://apacheignite.readme.io/docs/clustering" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
         .panel-body
             .col-sm-6
                 .settings-row
-                    +text('Name:', model + '.name', '"clusterName"', 'true', 'Input name', 'Grid name allows to indicate to what grid this particular grid instance belongs to')
+                    +text('Name:', `${model}.name`, '"clusterName"', 'true', 'Input name', 'Grid name allows to indicate to what grid this particular grid instance belongs to')
                 .settings-row
                     +caches(model, 'Select caches to start in cluster or add a new cache')
                 .settings-row
-                    +text-ip-address('Local host:', model + '.localHost', '"localHost"', 'true', '0.0.0.0',
+                    +text-ip-address('Local host:', `${model}.localHost`, '"localHost"', 'true', '0.0.0.0',
                         'System-wide local address or host for all Ignite components to bind to<br/>\
                         If not defined then Ignite tries to use local wildcard address<br/>\
                         That means that all services will be available on all network interfaces of the host machine')
                 .settings-row
-                    +dropdown('Discovery:', model + '.discovery.kind', '"discovery"', 'true', 'Choose discovery', 'discoveries',
+                    +dropdown('Discovery:', `${model}.discovery.kind`, '"discovery"', 'true', 'Choose discovery', 'discoveries',
                         'Discovery allows to discover remote nodes in grid\
                         <ul>\
                             <li>Static IPs - IP Finder which works only with pre configured list of IP addresses specified</li>\
@@ -55,22 +55,22 @@ include /app/helpers/jade/mixins.jade
                         </ul>')
                 .settings-row
                     .panel-details
-                        div(ng-if='#{modelDiscoveryKind} === "Cloud"')
-                            include ./general/discovery/cloud.jade
-                        div(ng-if='#{modelDiscoveryKind} === "GoogleStorage"')
-                            include ./general/discovery/google.jade
-                        div(ng-if='#{modelDiscoveryKind} === "Jdbc"')
-                            include ./general/discovery/jdbc.jade
-                        div(ng-if='#{modelDiscoveryKind} === "Multicast"')
-                            include ./general/discovery/multicast.jade
-                        div(ng-if='#{modelDiscoveryKind} === "S3"')
-                            include ./general/discovery/s3.jade
-                        div(ng-if='#{modelDiscoveryKind} === "SharedFs"')
-                            include ./general/discovery/shared.jade
-                        div(ng-if='#{modelDiscoveryKind} === "Vm"')
-                            include ./general/discovery/vm.jade
-                        div(ng-if='#{modelDiscoveryKind} === "ZooKeeper"')
-                            include ./general/discovery/zookeeper.jade
+                        div(ng-if=`${modelDiscoveryKind} === 'Cloud'`)
+                            include ./general/discovery/cloud
+                        div(ng-if=`${modelDiscoveryKind} === 'GoogleStorage'`)
+                            include ./general/discovery/google
+                        div(ng-if=`${modelDiscoveryKind} === 'Jdbc'`)
+                            include ./general/discovery/jdbc
+                        div(ng-if=`${modelDiscoveryKind} === 'Multicast'`)
+                            include ./general/discovery/multicast
+                        div(ng-if=`${modelDiscoveryKind} === 'S3'`)
+                            include ./general/discovery/s3
+                        div(ng-if=`${modelDiscoveryKind} === 'SharedFs'`)
+                            include ./general/discovery/shared
+                        div(ng-if=`${modelDiscoveryKind} === 'Vm'`)
+                            include ./general/discovery/vm
+                        div(ng-if=`${modelDiscoveryKind} === 'ZooKeeper'`)
+                            include ./general/discovery/zookeeper
             .col-sm-6
                 -var model = 'backupItem'
                 +preview-xml-java(model, 'clusterCaches', 'caches')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.pug
similarity index 78%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.pug
index 52fb21b619ae7..b127a4c7a13c3 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.pug
@@ -14,34 +14,34 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var discoveryKind = 'Cloud'
 -var required = 'backupItem.discovery.kind == "' + discoveryKind + '"'
 -var model = 'backupItem.discovery.Cloud'
--var regions = model + '.regions'
--var zones = model + '.zones'
+-var regions = `${model}.regions`
+-var zones = `${model}.zones`
 -var formRegions = 'discoveryCloudRegions'
 -var formZones = 'discoveryCloudZones'
 
 div
     .details-row
-        +text('Credential:', model + '.credential', '"credential"', 'false', 'Input cloud credential',
+        +text('Credential:', `${model}.credential`, '"credential"', 'false', 'Input cloud credential',
             'Credential that is used during authentication on the cloud<br/>\
             Depending on a cloud platform it can be a password or access key')
     .details-row
-        +text('Path to credential:', model + '.credentialPath', '"credentialPath"', 'false', 'Input pathto credential',
+        +text('Path to credential:', `${model}.credentialPath`, '"credentialPath"', 'false', 'Input pathto credential',
             'Path to a credential that is used during authentication on the cloud<br/>\
             Access key or private key should be stored in a plain or PEM file without a passphrase')
     .details-row
-        +text('Identity:', model + '.identity', '"' + discoveryKind + 'Identity"', required, 'Input identity',
+        +text('Identity:', `${model}.identity`, '"' + discoveryKind + 'Identity"', required, 'Input identity',
             'Identity that is used as a user name during a connection to the cloud<br/>\
             Depending on a cloud platform it can be an email address, user name, etc')
     .details-row
-        +text('Provider:', model + '.provider', '"' + discoveryKind + 'Provider"', required, 'Input provider', 'Cloud provider to use')
+        +text('Provider:', `${model}.provider`, '"' + discoveryKind + 'Provider"', required, 'Input provider', 'Cloud provider to use')
     .details-row
         -var form = formRegions;
-        +ignite-form-group(ng-model=regions ng-form=form)
+        +ignite-form-group(ng-model=`${regions}` ng-form=form)
             -var uniqueTip = 'Such region already exists!'
 
             ignite-form-field-label
@@ -53,19 +53,19 @@ div
             ignite-form-group-add(ng-click='group.add = [{}]')
                 | Add new region
 
-            .group-content(ng-if='#{regions}.length')
+            .group-content(ng-if=`${regions}.length`)
                 -var model = 'field.model';
                 -var name = '"edit" + $index'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = regions + '[$index] = ' + model
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${regions}[$index] = ${model}`
 
-                div(ng-repeat='model in #{regions} track by $index')
+                div(ng-repeat=`model in ${regions} track by $index`)
                     label.col-xs-12.col-sm-12.col-md-12(ng-init='field = {}')
                         .indexField
                             | {{ $index+1 }})
                         +table-remove-button(regions, 'Remove region')
                         span(ng-hide='field.edit')
-                            a.labelFormField(ng-click='field.edit = true; #{model} = model;') {{ model }}
+                            a.labelFormField(ng-click=`field.edit = true; ${model} = model;`) {{ model }}
                         span(ng-if='field.edit')
                             +table-text-field(name, model, regions, valid, save, 'Region name', false)
                                 +table-save-button(valid, save, false)
@@ -74,8 +74,8 @@ div
             .group-content(ng-repeat='field in group.add')
                 -var model = 'field.new';
                 -var name = '"new"'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = regions + '.push(' + model + ')'
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${regions}.push(${model})`
 
                 div
                     label.col-xs-12.col-sm-12.col-md-12
@@ -83,7 +83,7 @@ div
                             +table-save-button(valid, save, false)
                             +unique-feedback(name, uniqueTip)
 
-            .group-content-empty(ng-if='!(#{regions}.length) && !group.add.length')
+            .group-content-empty(ng-if=`!(${regions}.length) && !group.add.length`)
                 | Not defined
     .details-row
         -var form = formZones;
@@ -100,19 +100,19 @@ div
                 | Add new zone
 
             -var form = formZones;
-            .group-content(ng-if='#{zones}.length')
+            .group-content(ng-if=`${zones}.length`)
                 -var model = 'field.model';
                 -var name = '"edit" + $index'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = zones + '[$index] = ' + model
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${zones}[$index] = ${model}`
 
-                div(ng-repeat='model in #{zones} track by $index')
+                div(ng-repeat=`model in ${zones} track by $index`)
                     label.col-xs-12.col-sm-12.col-md-12(ng-init='field = {}')
                         .indexField
                             | {{ $index+1 }})
                         +table-remove-button(zones, 'Remove zone')
                         span(ng-hide='field.edit')
-                            a.labelFormField(ng-click='field.edit = true; #{model} = model;') {{ model }}
+                            a.labelFormField(ng-click=`field.edit = true; ${model} = model;`) {{ model }}
                         span(ng-if='field.edit')
                             +table-text-field(name, model, zones, valid, save, 'Zone name', false)
                                 +table-save-button(valid, save, false)
@@ -121,8 +121,8 @@ div
             .group-content(ng-repeat='field in group.add')
                 -var model = 'field.new';
                 -var name = '"new"'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = zones + '.push(' + model + ')'
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${zones}.push(${model})`
 
                 div
                     label.col-xs-12.col-sm-12.col-md-12
@@ -130,5 +130,5 @@ div
                             +table-save-button(valid, save, true)
                             +unique-feedback(name, uniqueTip)
 
-            .group-content-empty(ng-if='!(#{zones}.length) && !group.add.length')
+            .group-content-empty(ng-if=`!(${zones}.length) && !group.add.length`)
                 | Not defined
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.pug
similarity index 68%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.pug
index c12bd04c8f55c..929350e5f7ecd 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.pug
@@ -14,25 +14,25 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 
 -var discoveryKind = 'GoogleStorage'
--var required = 'backupItem.discovery.kind == "' + discoveryKind + '"'
+-var required = `backupItem.discovery.kind == '${discoveryKind}'`
 -var model = 'backupItem.discovery.GoogleStorage'
 
 div
     .details-row
-        +text('Project name:', model + '.projectName', '"' + discoveryKind + 'ProjectName"', required, 'Input project name', '' +
+        +text('Project name:', `${model}.projectName`, `'${discoveryKind}ProjectName'`, required, 'Input project name', '' +
             'Google Cloud Platforms project name<br/>\
             Usually this is an auto generated project number(ex. 208709979073) that can be found in "Overview" section of Google Developer Console')
     .details-row
-        +text('Bucket name:', model + '.bucketName', '"' + discoveryKind + 'BucketName"', required, 'Input bucket name',
+        +text('Bucket name:', `${model}.bucketName`, `'${discoveryKind}BucketName'`, required, 'Input bucket name',
             'Google Cloud Storage bucket name<br/>\
             If the bucket does not exist Ignite will automatically create it<br/>\
             However the name must be unique across whole Google Cloud Storage and Service Account Id must be authorized to perform this operation')
     .details-row
-        +text('Private key path:', model + '.serviceAccountP12FilePath', '"' + discoveryKind + 'ServiceAccountP12FilePath"', required, 'Input private key path',
+        +text('Private key path:', `${model}.serviceAccountP12FilePath`, `'${discoveryKind}ServiceAccountP12FilePath'`, required, 'Input private key path',
             'Full path to the private key in PKCS12 format of the Service Account')
     .details-row
-        +text('Account id:', model + '.serviceAccountId', '"' + discoveryKind + 'ServiceAccountId"', required, 'Input account id', 'Service account ID (typically an e-mail address)')
+        +text('Account id:', `${model}.serviceAccountId`, `'${discoveryKind}ServiceAccountId'`, required, 'Input account id', 'Service account ID (typically an e-mail address)')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.pug
similarity index 82%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.pug
index 4e4246db22f61..4e9a430308af1 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.pug
@@ -14,18 +14,18 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.Jdbc'
 -var required = 'backupItem.discovery.kind === "Jdbc"'
 
 div
     .details-row
-        +checkbox('DB schema should be initialized by Ignite', model + '.initSchema', '"initSchema"',
+        +checkbox('DB schema should be initialized by Ignite', `${model}.initSchema`, '"initSchema"',
             'Flag indicating whether DB schema should be initialized by Ignite or was explicitly created by user')
     .details-row
-        +text('Data source bean name:', model + '.dataSourceBean',
+        +text('Data source bean name:', `${model}.dataSourceBean`,
             '"dataSourceBean"', required, 'Input bean name', 'Name of the data source bean in Spring context')
     .details-row
-        +dialect('Dialect:', model + '.dialect', '"dialect"', required,
+        +dialect('Dialect:', `${model}.dialect`, '"dialect"', required,
             'Dialect of SQL implemented by a particular RDBMS:', 'Generic JDBC dialect', 'Choose JDBC dialect')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.pug
similarity index 79%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.pug
index 6a7e9faf80a51..f865fd37829c7 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var model = 'backupItem.discovery.Multicast'
@@ -22,25 +22,25 @@ include /app/helpers/jade/mixins.jade
 
 div
     .details-row
-        +text-ip-address('IP address:', model + '.multicastGroup', '"multicastGroup"', 'true', '228.1.2.4', 'IP address of multicast group')
+        +text-ip-address('IP address:', `${model}.multicastGroup`, '"multicastGroup"', 'true', '228.1.2.4', 'IP address of multicast group')
     .details-row
-        +number-min-max('Port number:', model + '.multicastPort', '"multicastPort"', 'true', '47400', '0', '65535', 'Port number which multicast messages are sent to')
+        +number-min-max('Port number:', `${model}.multicastPort`, '"multicastPort"', 'true', '47400', '0', '65535', 'Port number which multicast messages are sent to')
     .details-row
-        +number('Waits for reply:', model + '.responseWaitTime', '"responseWaitTime"', 'true', '500', '0',
+        +number('Waits for reply:', `${model}.responseWaitTime`, '"responseWaitTime"', 'true', '500', '0',
             'Time in milliseconds IP finder waits for reply to multicast address request')
     .details-row
-        +number('Attempts count:', model + '.addressRequestAttempts', '"addressRequestAttempts"', 'true', '2', '0',
+        +number('Attempts count:', `${model}.addressRequestAttempts`, '"addressRequestAttempts"', 'true', '2', '0',
             'Number of attempts to send multicast address request<br/>\
             IP finder re - sends request only in case if no reply for previous request is received')
     .details-row
-        +text-ip-address('Local address:', model + '.localAddress', '"localAddress"', 'true', '0.0.0.0',
+        +text-ip-address('Local address:', `${model}.localAddress`, '"localAddress"', 'true', '0.0.0.0',
             'Local host address used by this IP finder<br/>\
             If provided address is non - loopback then multicast socket is bound to this interface<br/>\
             If local address is not set or is any local address then IP finder creates multicast sockets for all found non - loopback addresses')
     .details-row
         -var form = 'discoveryMulticastAddresses';
 
-        +ignite-form-group(ng-model=addresses ng-form=form)
+        +ignite-form-group(ng-model=`${addresses}` ng-form=form)
             -var uniqueTip = 'Such IP address already exists!'
             -var ipAddressTip = 'Invalid IP address!'
 
@@ -61,13 +61,13 @@ div
             ignite-form-group-add(ng-click='group.add = [{}]')
                 | Add new address
 
-            .group-content(ng-if='#{addresses}.length')
+            .group-content(ng-if=`${addresses}.length`)
                 -var model = 'obj.model';
                 -var name = '"edit" + $index'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = addresses + '[$index] = ' + model
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${addresses}[$index] = ${model}`
 
-                div(ng-repeat='model in #{addresses} track by $index' ng-init='obj = {}')
+                div(ng-repeat=`model in ${addresses} track by $index` ng-init='obj = {}')
                     label.col-xs-12.col-sm-12.col-md-12
                         .indexField
                             | {{ $index+1 }})
@@ -77,7 +77,7 @@ div
                         +ignite-form-field-up(ng-if='!$first' ng-hide='field.edit' data-ng-model='model' data-models=addresses)
 
                         span(ng-hide='field.edit')
-                            a.labelFormField(ng-click='field.edit = true; #{model} = model;') {{ model }}
+                            a.labelFormField(ng-click=`field.edit = true; ${model} = model;`) {{ model }}
                         span(ng-if='field.edit')
                             +table-address-field(name, model, addresses, valid, save, false, true)
                                 +table-save-button(valid, save, false)
@@ -86,8 +86,8 @@ div
             .group-content(ng-repeat='field in group.add')
                 -var model = 'new';
                 -var name = '"new"'
-                -var valid = form + '[' + name + '].$valid'
-                -var save = addresses + '.push(' + model + ')'
+                -var valid = `${form}[${name}].$valid`
+                -var save = `${addresses}.push(${model})`
 
                 div
                     label.col-xs-12.col-sm-12.col-md-12
@@ -95,5 +95,5 @@ div
                             +table-save-button(valid, save, true)
                             +unique-feedback(name, uniqueTip)
 
-            .group-content-empty(ng-if='!(#{addresses}.length) && !group.add.length')
+            .group-content-empty(ng-if=`!(${addresses}.length) && !group.add.length`)
                 | Not defined
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
similarity index 79%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
index e5eaff342adbd..ed6e20e4c3c1a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var discoveryKind = 'S3'
--var required = 'backupItem.discovery.kind == "' + discoveryKind + '"'
+-var required = `backupItem.discovery.kind == '${discoveryKind}'`
 -var model = 'backupItem.discovery.S3'
 
 div
     .details-row
-        +text('Bucket name:', model + '.bucketName', '"' + discoveryKind + 'BucketName"', required, 'Input bucket name', 'Bucket name for IP finder')
+        +text('Bucket name:', `${model}.bucketName`, `${discoveryKind}BucketName'`, required, 'Input bucket name', 'Bucket name for IP finder')
     .details-row
         label Note, AWS credentials will be generated as stub
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.pug
similarity index 89%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.pug
index ddd9bfa9b1f05..8eb97368b0389 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.pug
@@ -14,10 +14,10 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.SharedFs'
 
 div
     .details-row
-        +text('File path:', model + '.path', '"path"', 'false', 'disco/tcp', 'Shared path')
+        +text('File path:', `${model}.path`, '"path"', 'false', 'disco/tcp', 'Shared path')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.pug
similarity index 83%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.pug
index 46ebae005833d..9205c7248b3fa 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.Vm'
 -var addresses = model + '.addresses'
 -var form = 'discoveryVmAddresses'
 
 .details-row
-    +ignite-form-group(ng-form=form ng-model=addresses)
+    +ignite-form-group(ng-form=form ng-model=`${addresses}`)
         -var uniqueTip = 'Such IP address already exists!'
 
         ignite-form-field-label
@@ -41,13 +41,13 @@ include /app/helpers/jade/mixins.jade
         ignite-form-group-add(ng-click='group.add = [{}]')
             | Add new address
 
-        .group-content(ng-if='#{addresses}.length')
+        .group-content(ng-if=`${addresses}.length`)
             -var model = 'obj.model';
             -var name = '"edit" + $index'
-            -var valid = form + '[' + name + '].$valid'
-            -var save = addresses + '[$index] = ' + model
+            -var valid = `${form}[${name}].$valid`
+            -var save = `${addresses}[$index] = ${model}`
 
-            div(ng-repeat='model in #{addresses} track by $index' ng-init='obj = {}')
+            div(ng-repeat=`model in ${addresses} track by $index` ng-init='obj = {}')
                 label.col-xs-12.col-sm-12.col-md-12
                     .indexField
                         | {{ $index+1 }})
@@ -57,7 +57,7 @@ include /app/helpers/jade/mixins.jade
                     +ignite-form-field-up(ng-if='!$first' ng-hide='field.edit' data-ng-model='model' data-models=addresses)
 
                     span(ng-hide='field.edit')
-                        a.labelFormField(ng-click='field.edit = true; #{model} = model;') {{ model }}
+                        a.labelFormField(ng-click=`field.edit = true; ${model} = model;`) {{ model }}
                     span(ng-if='field.edit')
                         +table-address-field(name, model, addresses, valid, save, false, true)
                             +table-save-button(valid, save, false)
@@ -66,8 +66,8 @@ include /app/helpers/jade/mixins.jade
         .group-content(ng-repeat='field in group.add')
             -var model = 'new';
             -var name = '"new"'
-            -var valid = form + '[' + name + '].$valid'
-            -var save = addresses + '.push(' + model + ')'
+            -var valid = `${form}[${name}].$valid`
+            -var save = `${addresses}.push(${model})`
 
             div
                 label.col-xs-12.col-sm-12.col-md-12
@@ -75,5 +75,5 @@ include /app/helpers/jade/mixins.jade
                         +table-save-button(valid, save, true)
                         +unique-feedback(name, uniqueTip)
 
-        .group-content-empty(id='addresses' ng-if='!(#{addresses}.length) && !group.add.length')
+        .group-content-empty(id='addresses' ng-if=`!(${addresses}.length) && !group.add.length`)
                 | Not defined
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.pug
similarity index 70%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.pug
index 48b1776db2c35..91548e4de99d6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.pug
@@ -14,25 +14,25 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var discoveryKind = 'ZooKeeper'
--var required = 'backupItem.discovery.kind == "' + discoveryKind + '"'
+-var required = `backupItem.discovery.kind == '${discoveryKind}'`
 -var model = 'backupItem.discovery.ZooKeeper'
--var modelRetryPolicyKind = model + '.retryPolicy.kind'
+-var modelRetryPolicyKind = `${model}.retryPolicy.kind`
 
 div
     .details-row
-        +java-class('Curator:', model + '.curator', '"curator"', 'true', 'false',
+        +java-class('Curator:', `${model}.curator`, '"curator"', 'true', 'false',
             'The Curator framework in use<br/>\
             By default generates curator of org.apache.curator. framework.imps.CuratorFrameworkImpl\
             class with configured connect string, retry policy, and default session and connection timeouts', required)
     .details-row
-        +text('Connect string:', model + '.zkConnectionString', '"' + discoveryKind + 'ConnectionString"', required, 'host:port[chroot][,host:port[chroot]]',
+        +text('Connect string:', `${model}.zkConnectionString`, `'${discoveryKind}ConnectionString'`, required, 'host:port[chroot][,host:port[chroot]]',
             'When "IGNITE_ZK_CONNECTION_STRING" system property is not configured this property will be used')
     .details-row
-        +dropdown('Retry policy:', model + '.retryPolicy.kind', '"retryPolicy"', 'true', 'Default',
+        +dropdown('Retry policy:', `${model}.retryPolicy.kind`, '"retryPolicy"', 'true', 'Default',
             '[\
                 {value: "ExponentialBackoff", label: "Exponential backoff"},\
                 {value: "BoundedExponentialBackoff", label: "Bounded exponential backoff"},\
@@ -54,32 +54,32 @@ div
                 <li>Custom - custom retry policy implementation</li>\
                 <li>Default - exponential backoff retry policy with configured base sleep time equal to 1000ms and max retry count equal to 10</li>\
             </ul>')
-    .details-row(ng-show='#{model}.retryPolicy.kind')
+    .details-row(ng-show=`${model}.retryPolicy.kind`)
         .panel-details
-                div(ng-show='#{modelRetryPolicyKind} === "ExponentialBackoff"')
-                    include ./zookeeper/retrypolicy/exponential-backoff.jade
-                div(ng-show='#{modelRetryPolicyKind} === "BoundedExponentialBackoff"')
-                    include ./zookeeper/retrypolicy/bounded-exponential-backoff.jade
-                div(ng-show='#{modelRetryPolicyKind} === "UntilElapsed"')
-                    include ./zookeeper/retrypolicy/until-elapsed.jade
-                div(ng-show='#{modelRetryPolicyKind} === "NTimes"')
-                    include ./zookeeper/retrypolicy/n-times.jade
-                div(ng-show='#{modelRetryPolicyKind} === "OneTime"')
-                    include ./zookeeper/retrypolicy/one-time.jade
-                div(ng-show='#{modelRetryPolicyKind} === "Forever"')
-                    include ./zookeeper/retrypolicy/forever.jade
-                div(ng-show='#{modelRetryPolicyKind} === "Custom"')
-                    include ./zookeeper/retrypolicy/custom.jade
+                div(ng-show=`${modelRetryPolicyKind} === 'ExponentialBackoff'`)
+                    include ./zookeeper/retrypolicy/exponential-backoff
+                div(ng-show=`${modelRetryPolicyKind} === 'BoundedExponentialBackoff'`)
+                    include ./zookeeper/retrypolicy/bounded-exponential-backoff
+                div(ng-show=`${modelRetryPolicyKind} === 'UntilElapsed'`)
+                    include ./zookeeper/retrypolicy/until-elapsed
+                div(ng-show=`${modelRetryPolicyKind} === 'NTimes'`)
+                    include ./zookeeper/retrypolicy/n-times
+                div(ng-show=`${modelRetryPolicyKind} === 'OneTime'`)
+                    include ./zookeeper/retrypolicy/one-time
+                div(ng-show=`${modelRetryPolicyKind} === 'Forever'`)
+                    include ./zookeeper/retrypolicy/forever
+                div(ng-show=`${modelRetryPolicyKind} === 'Custom'`)
+                    include ./zookeeper/retrypolicy/custom
     .details-row
         -var model = 'backupItem.discovery.ZooKeeper'
 
-        +text('Base path:', model + '.basePath', '"basePath"', 'false', '/services', 'Base path for service registration')
+        +text('Base path:', `${model}.basePath`, '"basePath"', 'false', '/services', 'Base path for service registration')
     .details-row
-        +text('Service name:', model + '.serviceName', '"serviceName"', 'false', 'ignite',
+        +text('Service name:', `${model}.serviceName`, '"serviceName"', 'false', 'ignite',
             'Service name to use, as defined by Curator&#39;s ServiceDiscovery recipe<br/>\
             In physical ZooKeeper terms, it represents the node under basePath, under which services will be registered')
     .details-row
-        +checkbox('Allow duplicate registrations', model + '.allowDuplicateRegistrations', '"allowDuplicateRegistrations"',
+        +checkbox('Allow duplicate registrations', `${model}.allowDuplicateRegistrations`, '"allowDuplicateRegistrations"',
             'Whether to register each node only once, or if duplicate registrations are allowed<br/>\
             Nodes will attempt to register themselves, plus those they know about<br/>\
             By default, duplicate registrations are not allowed, but you might want to set this property to <b>true</b> if you have multiple network interfaces or if you are facing troubles')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
similarity index 84%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
index d3c1f9fc88fbc..c71be9a1915cf 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.BoundedExponentialBackoff'
 
 div
     .details-row
-        +number('Base interval:', model + '.baseSleepTimeMs', '"beBaseSleepTimeMs"', 'true', '1000', '0', 'Initial amount of time in ms to wait between retries')
+        +number('Base interval:', `${model}.baseSleepTimeMs`, '"beBaseSleepTimeMs"', 'true', '1000', '0', 'Initial amount of time in ms to wait between retries')
     .details-row
-        +number('Max interval:', model + '.maxSleepTimeMs', '"beMaxSleepTimeMs"', 'true', 'Integer.MAX_VALUE', '0', 'Max time in ms to sleep on each retry')
+        +number('Max interval:', `${model}.maxSleepTimeMs`, '"beMaxSleepTimeMs"', 'true', 'Integer.MAX_VALUE', '0', 'Max time in ms to sleep on each retry')
     .details-row
-        +number-min-max('Max retries:', model + '.maxRetries', '"beMaxRetries"', 'true', '10', '0', '29', 'Max number of times to retry')
+        +number-min-max('Max retries:', `${model}.maxRetries`, '"beMaxRetries"', 'true', '10', '0', '29', 'Max number of times to retry')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.pug
similarity index 89%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.pug
index 5db89f5162e55..884a7d61bfd81 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.pug
@@ -14,11 +14,11 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy'
--var retry = model + '.Custom'
+-var retry = `${model}.Custom`
 -var required = 'backupItem.discovery.kind === "ZooKeeper" && backupItem.discovery.ZooKeeper.retryPolicy.kind === "Custom"'
 
 .details-row
-    +java-class('Class name:', retry + '.className', '"customClassName"', 'true', required, 'Custom retry policy implementation class name', required)
+    +java-class('Class name:', `${retry}.className`, '"customClassName"', 'true', required, 'Custom retry policy implementation class name', required)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
similarity index 83%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
index 0a072f73301f3..93b2709136c7b 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.ExponentialBackoff'
 
 div
     .details-row
-        +number('Base interval:', model + '.baseSleepTimeMs', '"expBaseSleepTimeMs"', 'true', '1000', '0', 'Initial amount of time in ms to wait between retries')
+        +number('Base interval:', `${model}.baseSleepTimeMs`, '"expBaseSleepTimeMs"', 'true', '1000', '0', 'Initial amount of time in ms to wait between retries')
     .details-row
-        +number-min-max('Max retries:', model + '.maxRetries', '"expMaxRetries"', 'true', '10', '0', '29', 'Max number of times to retry')
+        +number-min-max('Max retries:', `${model}.maxRetries`, '"expMaxRetries"', 'true', '10', '0', '29', 'Max number of times to retry')
     .details-row
-        +number('Max interval:', model + '.maxSleepMs', '"expMaxSleepMs"', 'true', 'Integer.MAX_VALUE', '0', 'Max time in ms to sleep on each retry')
+        +number('Max interval:', `${model}.maxSleepMs`, '"expMaxSleepMs"', 'true', 'Integer.MAX_VALUE', '0', 'Max time in ms to sleep on each retry')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.pug
index a7bfd20b42171..2d7d2e8397f2f 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.pug
@@ -14,9 +14,9 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.Forever'
 
 .details-row
-    +number('Interval:', model + '.retryIntervalMs', '"feRetryIntervalMs"', 'true', '1000', '0', 'Time in ms between retry attempts')
+    +number('Interval:', `${model}.retryIntervalMs`, '"feRetryIntervalMs"', 'true', '1000', '0', 'Time in ms between retry attempts')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.pug
similarity index 86%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.pug
index 1746232124ee8..fdaead1b4201d 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.pug
@@ -14,12 +14,12 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.NTimes'
 
 div
     .details-row
-        +number('Retries:', model + '.n', '"n"', 'true', '10', '0', 'Number of times to retry')
+        +number('Retries:', `${model}.n`, '"n"', 'true', '10', '0', 'Number of times to retry')
     .details-row
-        +number('Interval:', model + '.sleepMsBetweenRetries', '"ntSleepMsBetweenRetries"', 'true', '1000', '0', 'Time in ms between retry attempts')
+        +number('Interval:', `${model}.sleepMsBetweenRetries`, '"ntSleepMsBetweenRetries"', 'true', '1000', '0', 'Time in ms between retry attempts')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.pug
index 5a7e3fe28ccec..53fc9be265d88 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.pug
@@ -14,10 +14,10 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.OneTime'
 
 div
     .details-row
-        +number('Interval:', model + '.sleepMsBetweenRetry', '"oneSleepMsBetweenRetry"', 'true', '1000', '0', 'Time in ms to retry attempt')
+        +number('Interval:', `${model}.sleepMsBetweenRetry`, '"oneSleepMsBetweenRetry"', 'true', '1000', '0', 'Time in ms to retry attempt')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
similarity index 86%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
index 548211c4ed2a9..16ddfc5277d94 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
@@ -14,12 +14,12 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem.discovery.ZooKeeper.retryPolicy.UntilElapsed'
 
 div
     .details-row
-        +number('Total time:', model + '.maxElapsedTimeMs', '"ueMaxElapsedTimeMs"', 'true', '60000', '0', 'Total time in ms for execution of retry attempt')
+        +number('Total time:', `${model}.maxElapsedTimeMs`, '"ueMaxElapsedTimeMs"', 'true', '60000', '0', 'Total time in ms for execution of retry attempt')
     .details-row
-        +number('Interval:', model + '.sleepMsBetweenRetries', '"ueSleepMsBetweenRetries"', 'true', '1000', '0', 'Time in ms between retry attempts')
+        +number('Interval:', `${model}.sleepMsBetweenRetries`, '"ueSleepMsBetweenRetries"', 'true', '1000', '0', 'Time in ms between retry attempts')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.pug
similarity index 84%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.pug
index 25bd5b0193168..8c20a45ea1fbe 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.pug
@@ -14,25 +14,25 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'igfs'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label IGFS
         ignite-form-field-tooltip.tipLabel
             | IGFS (Ignite In-Memory File System) configurations assigned to cluster#[br]
             | #[a(href="https://apacheignite-fs.readme.io/docs/in-memory-file-system" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +dropdown-multiple('<span>IGFS:</span><a ui-sref="base.configuration.igfs({linkId: linkId()})"> (add)</a>',
-                        model + '.igfss', '"igfss"', true, 'Choose IGFS', 'No IGFS configured', 'igfss',
+                        `${model}.igfss`, '"igfss"', true, 'Choose IGFS', 'No IGFS configured', 'igfss',
                         'Select IGFS to start in cluster or add a new IGFS')
             .col-sm-6
                 +preview-xml-java(model, 'igfss', 'igfss')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.pug
similarity index 94%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.pug
index 9fa9fc9300cf5..b80f43065351d 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var model = 'backupItem'
 -var form = 'loadBalancing'
@@ -23,15 +23,15 @@ include /app/helpers/jade/mixins.jade
 -var loadProbeCustom = 'model.kind === "Adaptive" && model.Adaptive.loadProbe.kind === "Custom"'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Load balancing configuration
         ignite-form-field-tooltip.tipLabel
             | Load balancing component balances job distribution among cluster nodes#[br]
             | #[a(href="https://apacheignite.readme.io/docs/load-balancing" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row(ng-init='loadBalancingSpiTbl={type: "loadBalancingSpi", model: "loadBalancingSpi", focusId: "kind", ui: "load-balancing-table"}')
                     +ignite-form-group()
@@ -41,9 +41,9 @@ include /app/helpers/jade/mixins.jade
                             | Load balancing component balances job distribution among cluster nodes
                         ignite-form-group-add(ng-click='tableNewItem(loadBalancingSpiTbl)')
                             | Add load balancing configuration
-                        .group-content-empty(ng-if='!(#{loadBalancingSpi} && #{loadBalancingSpi}.length > 0)')
+                        .group-content-empty(ng-if=`!(${loadBalancingSpi} && ${loadBalancingSpi}.length > 0)`)
                             | Not defined
-                        .group-content(ng-show='#{loadBalancingSpi} && #{loadBalancingSpi}.length > 0' ng-repeat='model in #{loadBalancingSpi} track by $index')
+                        .group-content(ng-show=`${loadBalancingSpi} && ${loadBalancingSpi}.length > 0` ng-repeat=`model in ${loadBalancingSpi} track by $index`)
                             hr(ng-if='$index != 0')
                             .settings-row
                                 +dropdown-required-autofocus('Load balancing:', 'model.kind', '"loadBalancingKind" + $index', 'true', 'true', 'Choose load balancing SPI', '[\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger.pug
similarity index 80%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/logger.pug
index 983d81855cba6..f1351141210b8 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger.pug
@@ -14,21 +14,21 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'logger'
 -var model = 'backupItem.logger'
 -var kind = model + '.kind'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Logger configuration
         ignite-form-field-tooltip.tipLabel
             | Logging functionality used throughout the system
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +dropdown('Logger:', kind, '"logger"', 'true', 'Default',
@@ -53,14 +53,14 @@ include /app/helpers/jade/mixins.jade
                             <li>Custom - custom logger implementation</li>\
                             <li>Default - Apache Log4j if awailable on classpath or Java logger otherwise</li>\
                         </ul>')
-                .settings-row(ng-show='#{kind} && (#{kind} === "Log4j2" || #{kind} === "Log4j" || #{kind} === "Custom")')
+                .settings-row(ng-show=`${kind} && (${kind} === 'Log4j2' || ${kind} === 'Log4j' || ${kind} === 'Custom')`)
                     .panel-details
-                        div(ng-show='#{kind} === "Log4j2"')
-                            include ./logger/log4j2.jade
-                        div(ng-show='#{kind} === "Log4j"')
-                            include ./logger/log4j.jade
-                        div(ng-show='#{kind} === "Custom"')
-                            include ./logger/custom.jade
+                        div(ng-show=`${kind} === 'Log4j2'`)
+                            include ./logger/log4j2
+                        div(ng-show=`${kind} === 'Log4j'`)
+                            include ./logger/log4j
+                        div(ng-show=`${kind} === 'Custom'`)
+                            include ./logger/custom
             .col-sm-6
                 -var model = 'backupItem.logger'
                 +preview-xml-java(model, 'clusterLogger')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.pug
index 87d2b7d7e56b9..9852e94fbd0ac 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'logger'
 -var model = 'backupItem.logger.Custom'
@@ -22,4 +22,4 @@ include /app/helpers/jade/mixins.jade
 
 div
     .details-row
-        +java-class('Class:', model + '.class', '"customLogger"', 'true', required, 'Logger implementation class name', required)
+        +java-class('Class:', `${model}.class`, '"customLogger"', 'true', required, 'Logger implementation class name', required)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.pug
similarity index 87%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.pug
index 1eaa52988955b..fadaea48a4f6a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'logger'
 -var model = 'backupItem.logger.Log4j'
@@ -22,7 +22,7 @@ include /app/helpers/jade/mixins.jade
 
 div
     .details-row
-        +dropdown('Level:', model + '.level', '"log4jLevel"', 'true', 'Default',
+        +dropdown('Level:', `${model}.level`, '"log4jLevel"', 'true', 'Default',
             '[\
                 {value: "OFF", label: "OFF"},\
                 {value: "FATAL", label: "FATAL"},\
@@ -36,7 +36,7 @@ div
             ]',
             'Level for internal log4j implementation')
     .details-row
-        +dropdown-required('Logger configuration:', model + '.mode', '"log4jMode"', 'true', 'true', 'Choose logger mode',
+        +dropdown-required('Logger configuration:', `${model}.mode`, '"log4jMode"', 'true', 'true', 'Choose logger mode',
             '[\
                 {value: "Default", label: "Default"},\
                 {value: "Path", label: "Path"}\
@@ -47,4 +47,4 @@ div
                 <li>Path - path or URI to XML configuration</li>\
             </ul>')
     .details-row(ng-show=pathRequired)
-        +text('Path:', model + '.path', '"log4jPath"', pathRequired, 'Input path', 'Path or URI to XML configuration')
+        +text('Path:', `${model}.path`, '"log4jPath"', pathRequired, 'Input path', 'Path or URI to XML configuration')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.pug
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.pug
index 17df4811ff019..d90ca09dba2c6 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'logger'
 -var model = 'backupItem.logger.Log4j2'
@@ -22,7 +22,7 @@ include /app/helpers/jade/mixins.jade
 
 div
     .details-row
-        +dropdown('Level:', model + '.level', '"log4j2Level"', 'true', 'Default',
+        +dropdown('Level:', `${model}.level`, '"log4j2Level"', 'true', 'Default',
             '[\
                 {value: "OFF", label: "OFF"},\
                 {value: "FATAL", label: "FATAL"},\
@@ -36,4 +36,4 @@ div
             ]',
             'Level for internal log4j2 implementation')
     .details-row
-        +text('Path:', model + '.path', '"log4j2Path"', log4j2Required, 'Input path', 'Path or URI to XML configuration')
+        +text('Path:', `${model}.path`, '"log4j2Path"', log4j2Required, 'Input path', 'Path or URI to XML configuration')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.pug
similarity index 76%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.pug
index 9316e7962fbed..f9d29b5ac7fcd 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'marshaller'
 -var model = 'backupItem'
@@ -22,7 +22,7 @@ include /app/helpers/jade/mixins.jade
 -var optMarshaller = marshaller + '.OptimizedMarshaller'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Marshaller
         ignite-form-field-tooltip.tipLabel
@@ -31,8 +31,8 @@ include /app/helpers/jade/mixins.jade
             | By default BinaryMarshaller will be used#[br]
             | #[a(href="https://apacheignite.readme.io/docs/binary-marshaller" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +dropdown('Marshaller:', marshaller + '.kind', '"kind"', 'true', 'Choose marshaller',
@@ -48,13 +48,13 @@ include /app/helpers/jade/mixins.jade
                             <li>Not set - BinaryMarshaller serialize and deserialize all objects in the binary format</li>\
                         </ul>')
                     a.customize(
-                        ng-if='#{marshaller}.kind && #{marshaller}.kind === "OptimizedMarshaller"'
-                        ng-click='#{marshaller}.expanded = !#{marshaller}.expanded'
-                    ) {{ #{marshaller}.expanded ? "Hide settings" : "Show settings"}}
+                        ng-if=`${marshaller}.kind && ${marshaller}.kind === 'OptimizedMarshaller'`
+                        ng-click=`${marshaller}.expanded = !${marshaller}.expanded`
+                    ) {{ #{marshaller}.expanded ? 'Hide settings' : 'Show settings'}}
                 .settings-row
-                    .panel-details(ng-show='#{marshaller}.expanded && #{marshaller}.kind === "OptimizedMarshaller"')
+                    .panel-details(ng-show=`${marshaller}.expanded && ${marshaller}.kind === 'OptimizedMarshaller'`)
                         .details-row
-                            +number('Streams pool size:', optMarshaller + '.poolSize', '"poolSize"', 'true', '0', '0',
+                            +number('Streams pool size:', `${optMarshaller}.poolSize`, '"poolSize"', 'true', '0', '0',
                                 'Specifies size of cached object streams used by marshaller<br/>\
                                 Object streams are cached for performance reason to avoid costly recreation for every serialization routine<br/>\
                                 If 0 (default), pool is not used and each thread has its own cached object stream which it keeps reusing<br/>\
@@ -62,15 +62,15 @@ include /app/helpers/jade/mixins.jade
                                 Consider using pool in this case. This will limit number of streams that can be created and, therefore, decrease memory consumption<br/>\
                                 NOTE: Using streams pool can decrease performance since streams will be shared between different threads which will lead to more frequent context switching')
                         .details-row
-                            +checkbox('Require serializable', optMarshaller + '.requireSerializable', '"requireSerializable"',
+                            +checkbox('Require serializable', `${optMarshaller}.requireSerializable`, '"requireSerializable"',
                                 'Whether marshaller should require Serializable interface or not')
                 .settings-row
-                    +checkbox('Marshal local jobs', model + '.marshalLocalJobs', '"marshalLocalJobs"', 'If this flag is enabled, jobs mapped to local node will be marshalled as if it was remote node')
+                    +checkbox('Marshal local jobs', `${model}.marshalLocalJobs`, '"marshalLocalJobs"', 'If this flag is enabled, jobs mapped to local node will be marshalled as if it was remote node')
                 .settings-row
-                    +number('Keep alive time:', model + '.marshallerCacheKeepAliveTime', '"marshallerCacheKeepAliveTime"', 'true', '10000', '0',
+                    +number('Keep alive time:', `${model}.marshallerCacheKeepAliveTime`, '"marshallerCacheKeepAliveTime"', 'true', '10000', '0',
                         'Keep alive time of thread pool that is in charge of processing marshaller messages')
                 .settings-row
-                    +number('Pool size:', model + '.marshallerCacheThreadPoolSize', '"marshallerCacheThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
+                    +number('Pool size:', `${model}.marshallerCacheThreadPoolSize`, '"marshallerCacheThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
                         'Default size of thread pool that is in charge of processing marshaller messages')
             .col-sm-6
                 +preview-xml-java(model, 'clusterMarshaller')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.pug
similarity index 73%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.pug
index d5212a94402fe..6810e6b298704 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.pug
@@ -14,33 +14,33 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'metrics'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Metrics
         ignite-form-field-tooltip.tipLabel
             | Cluster runtime metrics settings
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Elapsed time:', model + '.metricsExpireTime', '"metricsExpireTime"', 'true', 'Long.MAX_VALUE', '1',
+                    +number('Elapsed time:', `${model}.metricsExpireTime`, '"metricsExpireTime"', 'true', 'Long.MAX_VALUE', '1',
                         'Time in milliseconds after which a certain metric value is considered expired')
                 .settings-row
-                    +number('History size:', model + '.metricsHistorySize', '"metricsHistorySize"', 'true', '10000', '1',
+                    +number('History size:', `${model}.metricsHistorySize`, '"metricsHistorySize"', 'true', '10000', '1',
                         'Number of metrics kept in history to compute totals and averages')
                 .settings-row
-                    +number('Log frequency:', model + '.metricsLogFrequency', '"metricsLogFrequency"', 'true', '60000', '0',
+                    +number('Log frequency:', `${model}.metricsLogFrequency`, '"metricsLogFrequency"', 'true', '60000', '0',
                         'Frequency of metrics log print out<br/>\ ' +
                         'When <b>0</b> log print of metrics is disabled')
                 .settings-row
-                    +number('Update frequency:', model + '.metricsUpdateFrequency', '"metricsUpdateFrequency"', 'true', '2000', '0',
+                    +number('Update frequency:', `${model}.metricsUpdateFrequency`, '"metricsUpdateFrequency"', 'true', '2000', '0',
                         'Job metrics update frequency in milliseconds\
                         <ul>\
                             <li>If set to -1 job metrics are never updated</li>\
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.pug
similarity index 83%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.pug
index 101a11c11caac..a69c7ad6844ba 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.pug
@@ -14,27 +14,27 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'odbcConfiguration'
 -var model = 'backupItem.odbc'
 -var enabled = model + '.odbcEnabled'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label ODBC configuration
         ignite-form-field-tooltip.tipLabel
             | ODBC server configuration#[br]
             | #[a(href="https://apacheignite.readme.io/docs/odbc-driver" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +checkbox('Enabled', enabled, '"odbcEnabled"', 'Flag indicating whether to configure ODBC configuration')
                 .settings-row
-                    +text-ip-address-with-port-range('ODBC endpoint address:', model + '.endpointAddress', '"endpointAddress"', enabled, '0.0.0.0:10800..10810',
+                    +text-ip-address-with-port-range('ODBC endpoint address:', `${model}.endpointAddress`, '"endpointAddress"', enabled, '0.0.0.0:10800..10810',
                         'ODBC endpoint address. <br/>\
                         The following address formats are permitted:\
                         <ul>\
@@ -43,6 +43,6 @@ include /app/helpers/jade/mixins.jade
                             <li>hostname:port_from..port_to - will use provided hostname and port range</li>\
                         </ul>')
                 .settings-row
-                    +number('Maximum open cursors', model + '.maxOpenCursors', '"maxOpenCursors"', enabled, '128', '1', 'Maximum number of opened cursors per connection')
+                    +number('Maximum open cursors', `${model}.maxOpenCursors`, '"maxOpenCursors"', enabled, '128', '1', 'Maximum number of opened cursors per connection')
             .col-sm-6
                 +preview-xml-java(model, 'clusterODBC')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.pug
similarity index 73%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.pug
index fbd979c954c05..dcb3b21c300a0 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'sslConfiguration'
 -var cluster = 'backupItem'
@@ -23,31 +23,31 @@ include /app/helpers/jade/mixins.jade
 -var trust = model + '.trustManagers'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label(id='sslConfiguration-title') SSL configuration
         ignite-form-field-tooltip.tipLabel
             | Settings for SSL configuration for creating a secure socket layer#[br]
             | #[a(href="https://apacheignite.readme.io/docs/ssltls" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +checkbox('Enabled', enabled, '"sslEnabled"', 'Flag indicating whether to configure SSL configuration')
                 .settings-row
-                    +text-options('Algorithm to create a key manager:', model + '.keyAlgorithm', '"keyAlgorithm"', '["SumX509", "X509"]', enabled, 'false', 'SumX509',
+                    +text-options('Algorithm to create a key manager:', `${model}.keyAlgorithm`, '"keyAlgorithm"', '["SumX509", "X509"]', enabled, 'false', 'SumX509',
                         'Sets key manager algorithm that will be used to create a key manager<br/>\
                         Notice that in most cased default value suites well, however, on Android platform this value need to be set to X509')
                 .settings-row
-                    +text-enabled('Key store file:', model + '.keyStoreFilePath', '"keyStoreFilePath"', enabled, enabled, 'Path to the key store file',
+                    +text-enabled('Key store file:', `${model}.keyStoreFilePath`, '"keyStoreFilePath"', enabled, enabled, 'Path to the key store file',
                         'Path to the key store file<br/>\
                         This is a mandatory parameter since ssl context could not be initialized without key manager')
                 .settings-row
-                    +text-options('Key store type:', model + '.keyStoreType', '"keyStoreType"', '["JKS", "PCKS11", "PCKS12"]', enabled, 'false', 'JKS',
+                    +text-options('Key store type:', `${model}.keyStoreType`, '"keyStoreType"', '["JKS", "PCKS11", "PCKS12"]', enabled, 'false', 'JKS',
                         'Key store type used in context initialization')
                 .settings-row
-                    +text-options('Protocol:', model + '.protocol', '"protocol"', '["TSL", "SSL"]', enabled, 'false', 'TSL', 'Protocol for secure transport')
+                    +text-options('Protocol:', `${model}.protocol`, '"protocol"', '["TSL", "SSL"]', enabled, 'false', 'TSL', 'Protocol for secure transport')
                 .settings-row
                     -var form = 'trustManagers'
 
@@ -58,29 +58,29 @@ include /app/helpers/jade/mixins.jade
                             | Trust managers
                         ignite-form-group-tooltip
                             | Pre-configured trust managers
-                        ignite-form-group-add(ng-show='#{enabled}' ng-click='(group.add = [{}])')
+                        ignite-form-group-add(ng-show=`${enabled}` ng-click='(group.add = [{}])')
                             | Add new trust manager
 
-                        .group-content(ng-if='#{trust}.length')
+                        .group-content(ng-if=`${trust}.length`)
                             -var model = 'obj.model';
                             -var name = '"edit" + $index'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = trust + '[$index] = ' + model
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${trust}[$index] = ${model}`
                 
                             div(ng-show=enabled)
-                                div(ng-repeat='model in #{trust} track by $index' ng-init='obj = {}')
+                                div(ng-repeat=`model in ${trust} track by $index` ng-init='obj = {}')
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .indexField
                                             | {{ $index+1 }})
                                         +table-remove-conditional-button(trust, enabled, 'Remove trust manager', 'model')
                                         span(ng-hide='field.edit')
-                                            a.labelFormField(ng-click='#{enabled} && (field.edit = true) && (#{model} = model)') {{ model }}
+                                            a.labelFormField(ng-click=`${enabled} && (field.edit = true) && (${model} = model)`) {{ model }}
                                         span(ng-if='field.edit')
                                             +table-java-class-field('Trust manager', name, model, trust, valid, save, false)
                                                 +table-save-button(valid, save, false)
                                                 +unique-feedback(name, uniqueTip)
                             div(ng-hide=enabled)
-                                div(ng-repeat='model in #{trust} track by $index')
+                                div(ng-repeat=`model in ${trust} track by $index`)
                                     label.col-xs-12.col-sm-12.col-md-12
                                         .labelFormField.labelField
                                             | {{ $index+1 }})
@@ -90,8 +90,8 @@ include /app/helpers/jade/mixins.jade
                         .group-content(ng-repeat='field in group.add')
                             -var model = 'new';
                             -var name = '"new"'
-                            -var valid = form + '[' + name + '].$valid'
-                            -var save = trust + '.push(' + model + ')'
+                            -var valid = `${form}[${name}].$valid`
+                            -var save = `${trust}.push(${model})`
  
                             div
                                 label.col-xs-12.col-sm-12.col-md-12
@@ -99,12 +99,12 @@ include /app/helpers/jade/mixins.jade
                                         +table-save-button(valid, save, true)
                                         +unique-feedback(name, uniqueTip)
 
-                        .group-content-empty(ng-if='!(#{trust}.length) && !group.add.length')
+                        .group-content-empty(ng-if=`!(${trust}.length) && !group.add.length`)
                             | Not defined
 
-                .settings-row(ng-show='!#{trust}.length')
-                    +text-enabled('Trust store file:', model + '.trustStoreFilePath', '"trustStoreFilePath"', enabled, 'false', 'Path to the trust store file', 'Path to the trust store file')
-                .settings-row(ng-show='!#{trust}.length')
-                    +text-options('Trust store type:', model + '.trustStoreType', '"trustStoreType"', '["JKS", "PCKS11", "PCKS12"]', enabled, 'false', 'JKS', 'Trust store type used in context initialization')
+                .settings-row(ng-show=`!${trust}.length`)
+                    +text-enabled('Trust store file:', `${model}.trustStoreFilePath`, '"trustStoreFilePath"', enabled, 'false', 'Path to the trust store file', 'Path to the trust store file')
+                .settings-row(ng-show=`!${trust}.length`)
+                    +text-options('Trust store type:', `${model}.trustStoreType`, '"trustStoreType"', '["JKS", "PCKS11", "PCKS12"]', enabled, 'false', 'JKS', 'Trust store type used in context initialization')
             .col-sm-6
                 +preview-xml-java(cluster, 'clusterSsl')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/swap.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/swap.pug
similarity index 73%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/swap.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/swap.pug
index 8071f3c282a6b..54b6db07a708b 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/swap.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/swap.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'swap'
 -var model = 'backupItem'
@@ -22,18 +22,18 @@ include /app/helpers/jade/mixins.jade
 -var fileSwapModel = swapModel + '.FileSwapSpaceSpi'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Swap
         ignite-form-field-tooltip.tipLabel
             | Settings for overflow data to disk if it cannot fit in memory#[br]
             | #[a(href="https://apacheignite.readme.io/docs/off-heap-memory#swap-space" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Swap space SPI:', swapModel + '.kind', '"swapSpaceSpi"', 'true', 'Choose swap SPI',
+                    +dropdown('Swap space SPI:', `${swapModel}.kind`, '"swapSpaceSpi"', 'true', 'Choose swap SPI',
                         '[\
                             {value: "FileSwapSpaceSpi", label: "File-based swap"},\
                             {value: undefined, label: "Not set"}\
@@ -45,27 +45,27 @@ include /app/helpers/jade/mixins.jade
                             <li>Not set - File-based swap space SPI with default configuration when it needed</li>\
                         </ul>')
                     a.customize(
-                        ng-if='#{swapModel}.kind'
-                        ng-click='#{swapModel}.expanded = !#{swapModel}.expanded'
-                    ) {{ #{swapModel}.expanded ? "Hide settings" : "Show settings"}}
+                        ng-if=`${swapModel}.kind`
+                        ng-click=`${swapModel}.expanded = !${swapModel}.expanded`
+                    ) {{ #{swapModel}.expanded ? 'Hide settings' : 'Show settings'}}
                 .settings-row
-                    .panel-details(ng-show='#{swapModel}.expanded && #{swapModel}.kind')
+                    .panel-details(ng-show=`${swapModel}.expanded && ${swapModel}.kind`)
                         .details-row
-                            +text('Base directory:', fileSwapModel + '.baseDirectory', '"baseDirectory"', 'false', 'swapspace',
+                            +text('Base directory:', `${fileSwapModel}.baseDirectory`, '"baseDirectory"', 'false', 'swapspace',
                                 'Base directory where to write files')
                         .details-row
-                            +number('Read stripe size:', fileSwapModel + '.readStripesNumber', '"readStripesNumber"', 'true', 'availableProcessors', '0',
+                            +number('Read stripe size:', `${fileSwapModel}.readStripesNumber`, '"readStripesNumber"', 'true', 'availableProcessors', '0',
                                 'Read stripe size defines number of file channels to be used concurrently')
                         .details-row
-                            +number-min-max-step('Maximum sparsity:', fileSwapModel + '.maximumSparsity', '"maximumSparsity"', 'true', '0.5', '0', '0.999', '0.05',
+                            +number-min-max-step('Maximum sparsity:', `${fileSwapModel}.maximumSparsity`, '"maximumSparsity"', 'true', '0.5', '0', '0.999', '0.05',
                                 'This property defines maximum acceptable wasted file space to whole file size ratio<br/>\
                                 When this ratio becomes higher than specified number compacting thread starts working')
                         .details-row
-                            +number('Max write queue size:', fileSwapModel + '.maxWriteQueueSize', '"maxWriteQueueSize"', 'true', '1024 * 1024', '0',
+                            +number('Max write queue size:', `${fileSwapModel}.maxWriteQueueSize`, '"maxWriteQueueSize"', 'true', '1024 * 1024', '0',
                                 'Max write queue size in bytes<br/>\
                                 If there are more values are waiting for being written to disk then specified size, SPI will block on store operation')
                         .details-row
-                            +number('Write buffer size:', fileSwapModel + '.writeBufferSize', '"writeBufferSize"', 'true', '64 * 1024', '0',
+                            +number('Write buffer size:', `${fileSwapModel}.writeBufferSize`, '"writeBufferSize"', 'true', '64 * 1024', '0',
                                 'Write buffer size in bytes<br/>\
                                 Write to disk occurs only when this buffer is full')
             .col-sm-6
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/thread.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/thread.pug
similarity index 68%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/thread.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/thread.pug
index d2bea86587db1..31dd333ebdf6d 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/thread.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/thread.pug
@@ -14,35 +14,35 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'pools'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Thread pools size
         ignite-form-field-tooltip.tipLabel
             | Settings for node thread pools
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Public:', model + '.publicThreadPoolSize', '"publicThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
+                    +number('Public:', `${model}.publicThreadPoolSize`, '"publicThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
                         'Thread pool that is in charge of processing ComputeJob, GridJobs and user messages sent to node')
                 .settings-row
-                    +number('System:', model + '.systemThreadPoolSize', '"systemThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
+                    +number('System:', `${model}.systemThreadPoolSize`, '"systemThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
                         'Thread pool that is in charge of processing internal system messages')
                 .settings-row
-                    +number('Management:', model + '.managementThreadPoolSize', '"managementThreadPoolSize"', 'true', '4', '1',
+                    +number('Management:', `${model}.managementThreadPoolSize`, '"managementThreadPoolSize"', 'true', '4', '1',
                         'Thread pool that is in charge of processing internal and Visor ComputeJob, GridJobs')
                 .settings-row
-                    +number('IGFS:', model + '.igfsThreadPoolSize', '"igfsThreadPoolSize"', 'true', 'availableProcessors', '1',
+                    +number('IGFS:', `${model}.igfsThreadPoolSize`, '"igfsThreadPoolSize"', 'true', 'availableProcessors', '1',
                         'Thread pool that is in charge of processing outgoing IGFS messages')
                 .settings-row
-                    +number('Rebalance:', model + '.rebalanceThreadPoolSize', '"rebalanceThreadPoolSize"', 'true', '1', '1',
+                    +number('Rebalance:', `${model}.rebalanceThreadPoolSize`, '"rebalanceThreadPoolSize"', 'true', '1', '1',
                         'Max count of threads can be used at rebalancing')
             .col-sm-6
                 +preview-xml-java(model, 'clusterPools')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/time.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/time.pug
similarity index 73%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/time.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/time.pug
index bcb1cb71f5b53..da781baef08ed 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/time.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/time.pug
@@ -14,34 +14,34 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'time'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Time configuration
         ignite-form-field-tooltip.tipLabel
             | Time settings for CLOCK write ordering mode
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Samples size:', model + '.clockSyncSamples', '"clockSyncSamples"', 'true', '8', '0',
+                    +number('Samples size:', `${model}.clockSyncSamples`, '"clockSyncSamples"', 'true', '8', '0',
                         'Number of samples used to synchronize clocks between different nodes<br/>\
                         Clock synchronization is used for cache version assignment in CLOCK order mode')
                 .settings-row
-                    +number('Frequency:', model + '.clockSyncFrequency', '"clockSyncFrequency"', 'true', '120000', '0',
+                    +number('Frequency:', `${model}.clockSyncFrequency`, '"clockSyncFrequency"', 'true', '120000', '0',
                         'Frequency at which clock is synchronized between nodes, in milliseconds<br/>\
                         Clock synchronization is used for cache version assignment in CLOCK order mode')
                 .settings-row
-                    +number-min-max('Port base:', model + '.timeServerPortBase', '"timeServerPortBase"', 'true', '31100', '0', '65535',
+                    +number-min-max('Port base:', `${model}.timeServerPortBase`, '"timeServerPortBase"', 'true', '31100', '0', '65535',
                         'Time server provides clock synchronization between nodes<br/>\
                         Base UPD port number for grid time server. Time server will be started on one of free ports in range')
                 .settings-row
-                    +number('Port range:', model + '.timeServerPortRange', '"timeServerPortRange"', 'true', '100', '1', 'Time server port range')
+                    +number('Port range:', `${model}.timeServerPortRange`, '"timeServerPortRange"', 'true', '100', '1', 'Time server port range')
             .col-sm-6
                 +preview-xml-java(model, 'clusterTime')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.jade b/modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.pug
similarity index 78%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.jade
rename to modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.pug
index 6a18bb0e6a6f2..f60589f29ef0d 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.pug
@@ -14,24 +14,24 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'transactions'
 -var model = 'backupItem.transactionConfiguration'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Transactions
         ignite-form-field-tooltip.tipLabel
             | Settings for transactions#[br]
             | #[a(href="https://apacheignite.readme.io/docs/transactions" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +dropdown('Concurrency:', model + '.defaultTxConcurrency', '"defaultTxConcurrency"', 'true', 'PESSIMISTIC',
+                    +dropdown('Concurrency:', `${model}.defaultTxConcurrency`, '"defaultTxConcurrency"', 'true', 'PESSIMISTIC',
                         '[\
                             {value: "OPTIMISTIC", label: "OPTIMISTIC"},\
                             {value: "PESSIMISTIC", label: "PESSIMISTIC"}\
@@ -42,7 +42,7 @@ include /app/helpers/jade/mixins.jade
                             <li>PESSIMISTIC - A lock is acquired on all cache operations with exception of read operations in READ_COMMITTED mode</li>\
                         </ul>')
                 .settings-row
-                    +dropdown('Isolation:', model + '.defaultTxIsolation', '"defaultTxIsolation"', 'true', 'REPEATABLE_READ',
+                    +dropdown('Isolation:', `${model}.defaultTxIsolation`, '"defaultTxIsolation"', 'true', 'REPEATABLE_READ',
                         '[\
                             {value: "READ_COMMITTED", label: "READ_COMMITTED"},\
                             {value: "REPEATABLE_READ", label: "REPEATABLE_READ"},\
@@ -55,15 +55,15 @@ include /app/helpers/jade/mixins.jade
                             <li>SERIALIZABLE - All transactions occur in a completely isolated fashion, as if all transactions in the system had executed serially, one after the other.</li>\
                         </ul>')
                 .settings-row
-                    +number('Default timeout:', model + '.defaultTxTimeout', '"defaultTxTimeout"', 'true', '0', '0', 'Default transaction timeout')
+                    +number('Default timeout:', `${model}.defaultTxTimeout`, '"defaultTxTimeout"', 'true', '0', '0', 'Default transaction timeout')
                 .settings-row
-                    +number('Pessimistic log cleanup delay:', model + '.pessimisticTxLogLinger', '"pessimisticTxLogLinger"', 'true', '10000', '0',
+                    +number('Pessimistic log cleanup delay:', `${model}.pessimisticTxLogLinger`, '"pessimisticTxLogLinger"', 'true', '10000', '0',
                         'Delay, in milliseconds, after which pessimistic recovery entries will be cleaned up for failed node')
                 .settings-row
-                    +number('Pessimistic log size:', model + '.pessimisticTxLogSize', '"pessimisticTxLogSize"', 'true', '0', '0',
+                    +number('Pessimistic log size:', `${model}.pessimisticTxLogSize`, '"pessimisticTxLogSize"', 'true', '0', '0',
                         'Size of pessimistic transactions log stored on node in order to recover transaction commit if originating node has left grid before it has sent all messages to transaction nodes')
                 .settings-row
-                    +java-class('Manager factory:', model + '.txManagerFactory', '"txManagerFactory"', 'true', 'false',
+                    +java-class('Manager factory:', `${model}.txManagerFactory`, '"txManagerFactory"', 'true', 'false',
                         'Class name of transaction manager factory for integration with JEE app servers')
             .col-sm-6
                 +preview-xml-java(model, 'clusterTransactions')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/domains/general.jade b/modules/web-console/frontend/app/modules/states/configuration/domains/general.pug
similarity index 79%
rename from modules/web-console/frontend/app/modules/states/configuration/domains/general.jade
rename to modules/web-console/frontend/app/modules/states/configuration/domains/general.pug
index 40759e30c0748..6432adff97dcb 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/domains/general.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/domains/general.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var model = 'backupItem'
@@ -28,25 +28,25 @@ include /app/helpers/jade/mixins.jade
             | #[a(href="https://apacheignite.readme.io/docs/cache-queries" target="_blank") More info about query configuration]#[br]
             | #[a(href="https://apacheignite.readme.io/docs/persistent-store" target="_blank") More info about store]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
         .panel-body
             .col-sm-6
                 .settings-row
-                    +checkbox('Generate POJO classes', model + '.generatePojo', '"generatePojo"', 'If selected then POJO classes will be generated from database tables')
+                    +checkbox('Generate POJO classes', `${model}.generatePojo`, '"generatePojo"', 'If selected then POJO classes will be generated from database tables')
                 .settings-row
                     +caches(model, 'Select caches to associate domain model with cache')
                 .settings-row
-                    +dropdown-required('Query metadata:', model + '.queryMetadata', '"queryMetadata"', 'true', 'true', '', 'queryMetadataVariants',
+                    +dropdown-required('Query metadata:', `${model}.queryMetadata`, '"queryMetadata"', 'true', 'true', '', 'queryMetadataVariants',
                         'Query metadata configured with:\
                         <ul>\
                             <li>Java annotations like @QuerySqlField</li>\
                             <li>Configuration via QueryEntity class</li>\
                         </ul>')
-                -var generatePojo = model + '.generatePojo'
+                -var generatePojo = `${model}.generatePojo`
                 .settings-row
-                    +java-class-typeahead('Key type:', model + '.keyType', '"keyType"', 'javaBuiltInClasses', 'true', 'true', '{{ ' + generatePojo + ' ? "Full class name for Key" : "Key type name" }}', 'Key class used to store key in cache', generatePojo)
+                    +java-class-typeahead('Key type:', `${model}.keyType`, '"keyType"', 'javaBuiltInClasses', 'true', 'true', '{{ ' + generatePojo + ' ? "Full class name for Key" : "Key type name" }}', 'Key class used to store key in cache', generatePojo)
                 .settings-row
-                    +java-class-autofocus-placholder('Value type:', model + '.valueType', '"valueType"', 'true', 'true', 'false', '{{ ' + generatePojo +' ? "Enter fully qualified class name" : "Value type name" }}', 'Value class used to store value in cache', generatePojo)
+                    +java-class-autofocus-placholder('Value type:', `${model}.valueType`, '"valueType"', 'true', 'true', 'false', '{{ ' + generatePojo +' ? "Enter fully qualified class name" : "Value type name" }}', 'Value class used to store value in cache', generatePojo)
 
             .col-sm-6
                 +preview-xml-java(model, 'domainModelGeneral')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/domains/query.jade b/modules/web-console/frontend/app/modules/states/configuration/domains/query.pug
similarity index 77%
rename from modules/web-console/frontend/app/modules/states/configuration/domains/query.jade
rename to modules/web-console/frontend/app/modules/states/configuration/domains/query.pug
index 8ed7b6839c2c7..a057f59abbb2e 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/domains/query.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/domains/query.pug
@@ -14,42 +14,42 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'query'
 -var model = 'backupItem'
--var queryFields = model + '.fields'
--var queryAliases = model + '.aliases'
--var queryIndexes = model + '.indexes'
+-var queryFields = `${model}.fields`
+-var queryAliases = `${model}.aliases`
+-var queryIndexes = `${model}.indexes`
 -var queryFieldsForm = 'queryFields'
 -var queryAliasesForm = 'queryAliases'
 -var queryIndexesForm = 'queryIndexes'
 
 // LEGACY mixin for LEGACY index fields table.
 mixin table-index-item-edit(prefix, index, sortAvailable, idAddition)
-    -var fieldName = prefix + 'FieldName'
-    -var direction = prefix + 'Direction'
+    -var fieldName = `${prefix}FieldName`
+    -var direction = `${prefix}Direction`
 
-    -var fieldNameModel = 'indexesTbl.' + fieldName
-    -var directionModel = 'indexesTbl.' + direction
+    -var fieldNameModel = `indexesTbl.${fieldName}`
+    -var directionModel = `indexesTbl.${direction}`
 
-    -var btnVisible = 'tableIndexItemSaveVisible(indexesTbl, ' + index + ')'
-    -var btnSave = 'tableIndexItemSave(indexesTbl, itemIndex, ' + index + ')'
-    -var btnVisibleAndSave = btnVisible + ' && ' + btnSave
+    -var btnVisible = `tableIndexItemSaveVisible(indexesTbl, ${index})`
+    -var btnSave = `tableIndexItemSave(indexesTbl, itemIndex, ${index})`
+    -var btnVisibleAndSave = `${btnVisible} && ${btnSave}`
 
     div(ng-if=sortAvailable)
         .col-xs-8.col-sm-8.col-md-8
             label.fieldSep /
             .input-tip
-                button.select-toggle.form-control(id='{{::"#{fieldName}" + #{idAddition}}}' ignite-on-enter-focus-move='{{::"#{direction}S" + #{idAddition}}}' ng-model=fieldNameModel placeholder='{{fields("#{prefix}", #{fieldNameModel}).length > 0 ? "Choose field" : "No fields configured"}}' bs-select bs-options='item.value as item.label for item in fields("#{prefix}", #{fieldNameModel})' ng-disabled='fields("#{prefix}", #{fieldNameModel}).length === 0' ignite-on-escape='tableReset(false)' tabindex='0')
+                button.select-toggle.form-control(id=`{{::'${fieldName}' + ${idAddition}}}` ignite-on-enter-focus-move=`{{::'${direction}S' + ${idAddition}}}` ng-model=fieldNameModel placeholder=`{{fields('${prefix}', ${fieldNameModel}).length > 0 ? 'Choose field' : 'No fields configured'}}` bs-select bs-options=`item.value as item.label for item in fields('${prefix}', ${fieldNameModel})` ng-disabled=`fields('${prefix}', ${fieldNameModel}).length === 0` ignite-on-escape='tableReset(false)' tabindex='0')
         .col-xs-4.col-sm-4.col-md-4
             +btn-save(btnVisible, btnSave)
             .input-tip
-                button.select-toggle.form-control(id='{{::"#{direction}" + #{idAddition}}}' ng-model=directionModel bs-select bs-options='item.value as item.label for item in {{sortDirections}}' ignite-on-enter=btnVisibleAndSave ignite-on-escape='tableReset(false)' tabindex='0')
-    .col-xs-12(ng-if='!(#{sortAvailable})')
+                button.select-toggle.form-control(id=`{{::'${direction}' + ${idAddition}}}` ng-model=directionModel bs-select bs-options='item.value as item.label for item in {{sortDirections}}' ignite-on-enter=btnVisibleAndSave ignite-on-escape='tableReset(false)' tabindex='0')
+    .col-xs-12(ng-if=`!(${sortAvailable})`)
         +btn-save(btnVisible, btnSave)
         .input-tip
-            button.select-toggle.form-control(id='{{::"#{fieldName}" + #{idAddition}}}' ng-model=fieldNameModel placeholder='{{fields("#{prefix}", #{fieldNameModel}).length > 0 ? "Choose index field" : "No fields configured"}}' bs-select bs-options='item.value as item.label for item in fields("#{prefix}", #{fieldNameModel})' ng-disabled='fields("#{prefix}", #{fieldNameModel}).length === 0' ignite-on-escape='tableReset(false)' tabindex='0')
+            button.select-toggle.form-control(id=`{{::'${fieldName}' + ${idAddition}}}` ng-model=fieldNameModel placeholder=`{{fields('${prefix}', ${fieldNameModel}).length > 0 ? 'Choose index field' : 'No fields configured'}}` bs-select bs-options=`item.value as item.label for item in fields('${prefix}', ${fieldNameModel})` ng-disabled=`fields('${prefix}', ${fieldNameModel}).length === 0` ignite-on-escape='tableReset(false)' tabindex='0')
 
 .panel.panel-default(ng-form=form novalidate)
     .panel-heading(bs-collapse-toggle)
@@ -57,28 +57,28 @@ mixin table-index-item-edit(prefix, index, sortAvailable, idAddition)
         label(id='query-title') Domain model for SQL query
         ignite-form-field-tooltip.tipLabel
             | Domain model properties for fields queries#[br]
-            | #[a(href="https://apacheignite.readme.io/docs/cache-queries" target="_blank") More info]
+            | #[a(href='https://apacheignite.readme.io/docs/cache-queries' target='_blank') More info]
         ignite-form-revert
     .panel-collapse(role='tabpanel' bs-collapse-target id='query')
         .panel-body
             .col-sm-6
-                .content-not-available(ng-if='#{model}.queryMetadata === "Annotations"')
+                .content-not-available(ng-if=`${model}.queryMetadata === 'Annotations'`)
                     label Not available for annotated types
-                div(ng-if='#{model}.queryMetadata === "Configuration"')
+                div(ng-if=`${model}.queryMetadata === 'Configuration'`)
                     .settings-row
-                        +ignite-form-group(ng-model='#{queryFields}' ng-form='#{queryFieldsForm}')
+                        +ignite-form-group(ng-model=queryFields ng-form=queryFieldsForm)
                             ignite-form-field-label(id='queryFields')
                                 | Fields
                             ignite-form-group-tooltip
                                 | Collection of name-to-type mappings to be queried, in addition to indexed fields
                             ignite-form-group-add(ng-click='tableNewItem(queryFieldsTbl)')
                                 | Add field to query
-                            .group-content-empty(ng-if='!((#{queryFields} && #{queryFields}.length > 0) || tableNewItemActive(queryFieldsTbl))')
+                            .group-content-empty(ng-if=`!((${queryFields} && ${queryFields}.length > 0) || tableNewItemActive(queryFieldsTbl))`)
                                 | Not defined
-                            .group-content(ng-show='(#{queryFields} && #{queryFields}.length > 0) || tableNewItemActive(queryFieldsTbl)')
+                            .group-content(ng-show=`(${queryFields} && ${queryFields}.length > 0) || tableNewItemActive(queryFieldsTbl)`)
                                 table.links-edit(id='fields' st-table=queryFields)
                                     tbody
-                                        tr(ng-repeat='item in #{queryFields} track by $index')
+                                        tr(ng-repeat=`item in ${queryFields} track by $index`)
                                             td.col-sm-12(ng-hide='tableEditing(queryFieldsTbl, $index)')
                                                 a.labelFormField(ng-click='tableStartEdit(backupItem, queryFieldsTbl, $index)') {{item.name}}  / {{item.className}}
                                                 +btn-remove('tableRemove(backupItem, queryFieldsTbl, $index)', '"Remove path"')
@@ -89,7 +89,7 @@ mixin table-index-item-edit(prefix, index, sortAvailable, idAddition)
                                             td.col-sm-12
                                                 +table-pair-edit('queryFieldsTbl', 'new', 'Field name', 'Field full class name', false, true, '{{::queryFieldsTbl.focusId + $index}}', '-1', '/')
                     .settings-row
-                        +ignite-form-group(ng-model='#{queryAliases}' ng-form='#{queryAliasesForm}')
+                        +ignite-form-group(ng-model=queryAliases ng-form=queryAliasesForm)
                             ignite-form-field-label
                                 | Aliases
                             ignite-form-group-tooltip
@@ -97,12 +97,12 @@ mixin table-index-item-edit(prefix, index, sortAvailable, idAddition)
                                 | For example: "parent.name" as "parentName"
                             ignite-form-group-add(ng-click='tableNewItem(aliasesTbl)')
                                 | Add alias to query
-                            .group-content-empty(ng-if='!((#{queryAliases} && #{queryAliases}.length > 0) || tableNewItemActive(aliasesTbl))')
+                            .group-content-empty(ng-if=`!((${queryAliases} && ${queryAliases}.length > 0) || tableNewItemActive(aliasesTbl))`)
                                 | Not defined
-                            .group-content(ng-show='(#{queryAliases} && #{queryAliases}.length > 0) || tableNewItemActive(aliasesTbl)')
+                            .group-content(ng-show=`(${queryAliases} && ${queryAliases}.length > 0) || tableNewItemActive(aliasesTbl)`)
                                 table.links-edit(id='aliases' st-table=queryAliases)
                                     tbody
-                                        tr(ng-repeat='item in #{queryAliases} track by $index')
+                                        tr(ng-repeat=`item in ${queryAliases} track by $index`)
                                             td.col-sm-12(ng-hide='tableEditing(aliasesTbl, $index)')
                                                 a.labelFormField(ng-click='tableStartEdit(backupItem, aliasesTbl, $index)') {{item.field}} &rarr; {{item.alias}}
                                                 +btn-remove('tableRemove(backupItem, aliasesTbl, $index)', '"Remove alias"')
@@ -113,21 +113,21 @@ mixin table-index-item-edit(prefix, index, sortAvailable, idAddition)
                                             td.col-sm-12
                                                 +table-pair-edit('aliasesTbl', 'new', 'Field name', 'Field Alias', false, false, '{{::aliasesTbl.focusId + $index}}', '-1', '&rarr;')
                     .settings-row(ng-init='indexesTbl={type: "table-indexes", model: "indexes", focusId: "IndexName", ui: "table-indexes"}')
-                        +ignite-form-group(ng-model='#{queryIndexes}' ng-form='#{queryIndexesForm}')
+                        +ignite-form-group(ng-model=queryIndexes ng-form=queryIndexesForm)
                             ignite-form-field-label
                                 | Indexes
                             ignite-form-group-tooltip
                                 | Collection of indexes
                             ignite-form-group-add(ng-click='tableNewItem(indexesTbl)')
                                 | Add new index
-                            .group-content-empty(id='indexes-add' ng-show='!((#{queryIndexes} && #{queryIndexes}.length > 0) || tableNewItemActive(indexesTbl))')
+                            .group-content-empty(id='indexes-add' ng-show=`!((${queryIndexes} && ${queryIndexes}.length > 0) || tableNewItemActive(indexesTbl))`)
                                 | Not defined
-                            .group-content(ng-show='(#{queryIndexes} && #{queryIndexes}.length > 0) || tableNewItemActive(indexesTbl)')
+                            .group-content(ng-show=`(${queryIndexes} && ${queryIndexes}.length > 0) || tableNewItemActive(indexesTbl)`)
                                 -var btnVisibleAndSave = 'tableIndexSaveVisible(indexesTbl, $index) && tableIndexSave(indexesTbl, $index)'
 
                                 table.links-edit(st-table=queryIndexes ng-init='newDirection = false')
                                     tbody
-                                        tr(ng-repeat='item in #{queryIndexes} track by $index')
+                                        tr(ng-repeat=`item in ${queryIndexes} track by $index`)
                                             td
                                                 .col-sm-12(ng-hide='tableEditing(indexesTbl, $index)')
                                                     a.labelFormField(id='indexes{{$index}}' ng-click='tableStartEdit(backupItem, indexesTbl, $index)') {{$index + 1}}) {{item.name}} [{{item.indexType}}]
diff --git a/modules/web-console/frontend/app/modules/states/configuration/domains/store.jade b/modules/web-console/frontend/app/modules/states/configuration/domains/store.pug
similarity index 74%
rename from modules/web-console/frontend/app/modules/states/configuration/domains/store.jade
rename to modules/web-console/frontend/app/modules/states/configuration/domains/store.pug
index f5d6b7d075e5f..4dee98677950a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/domains/store.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/domains/store.pug
@@ -14,63 +14,63 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'store'
 -var model = 'backupItem'
--var keyFields = model + '.keyFields'
--var valueFields = model + '.valueFields'
+-var keyFields = `${model}.keyFields`
+-var valueFields = `${model}.valueFields`
 -var keyFieldsForm = 'storeKeyFields'
 -var valueFieldsForm = 'storeValueFields'
 
 //- LEGACY mixin for LEGACY db fields tables.
 mixin table-db-field-edit(tbl, prefix, focusId, index)
-    -var databaseName = prefix + 'DatabaseFieldName'
-    -var databaseType = prefix + 'DatabaseFieldType'
-    -var javaName = prefix + 'JavaFieldName'
-    -var javaType = prefix + 'JavaFieldType'
+    -var databaseName = `${prefix}DatabaseFieldName`
+    -var databaseType = `${prefix}DatabaseFieldType`
+    -var javaName = `${prefix}JavaFieldName`
+    -var javaType = `${prefix}JavaFieldType`
 
-    -var databaseNameModel = tbl + '.' + databaseName
-    -var databaseTypeModel = tbl + '.' + databaseType
-    -var javaNameModel = tbl + '.' + javaName
-    -var javaTypeModel = tbl + '.' + javaType
+    -var databaseNameModel = `${tbl}.${databaseName}`
+    -var databaseTypeModel = `${tbl}.${databaseType}`
+    -var javaNameModel = `${tbl}.${javaName}`
+    -var javaTypeModel = `${tbl}.${javaType}`
 
-    -var databaseNameId = databaseName + focusId
-    -var databaseTypeId = databaseType + focusId
-    -var javaNameId = javaName + focusId
-    -var javaTypeId = javaType + focusId
+    -var databaseNameId = `${databaseName}${focusId}`
+    -var databaseTypeId = `${databaseType}${focusId}`
+    -var javaNameId = `${javaName}${focusId}`
+    -var javaTypeId = `${javaType}${focusId}`
 
     .col-xs-3.col-sm-3.col-md-3
         .fieldSep /
         .input-tip
-            input.form-control(id=databaseNameId ignite-on-enter-focus-move=databaseTypeId type='text' ng-model=databaseNameModel placeholder='DB name' ignite-on-enter='#{javaNameModel} = #{javaNameModel} ? #{javaNameModel} : #{databaseNameModel}' ignite-on-escape='tableReset(false)')
+            input.form-control(id=databaseNameId ignite-on-enter-focus-move=databaseTypeId type='text' ng-model=databaseNameModel placeholder='DB name' ignite-on-enter=`${javaNameModel} = ${javaNameModel} ? ${javaNameModel} : ${databaseNameModel}` ignite-on-escape='tableReset(false)')
     .col-xs-3.col-sm-3.col-md-3
         .fieldSep /
         .input-tip
-            button.select-toggle.form-control(id=databaseTypeId ignite-on-enter-focus-move=javaNameId ng-model=databaseTypeModel data-placeholder='DB type' ng-class='{placeholder: !#{databaseTypeModel}}' bs-select bs-options='item.value as item.label for item in {{supportedJdbcTypes}}' ignite-on-escape='tableReset(false)' tabindex='0')
+            button.select-toggle.form-control(id=databaseTypeId ignite-on-enter-focus-move=javaNameId ng-model=databaseTypeModel data-placeholder='DB type' ng-class=`{placeholder: !${databaseTypeModel}}` bs-select bs-options='item.value as item.label for item in {{supportedJdbcTypes}}' ignite-on-escape='tableReset(false)' tabindex='0')
     .col-xs-3.col-sm-3.col-md-3
         .fieldSep /
         .input-tip
             input.form-control(id=javaNameId ignite-on-enter-focus-move=javaTypeId type='text' ng-model=javaNameModel placeholder='Java name' ignite-on-escape='tableReset(false)')
     .col-xs-3.col-sm-3.col-md-3
-        -var btnVisible = 'tableDbFieldSaveVisible(' + tbl + ', ' + index +')'
-        -var btnSave = 'tableDbFieldSave(' + tbl + ', ' + index +')'
-        -var btnVisibleAndSave = btnVisible + ' && ' + btnSave
+        -var btnVisible = `tableDbFieldSaveVisible(${tbl}, ${index})`
+        -var btnSave = `tableDbFieldSave(${tbl}, ${index})`
+        -var btnVisibleAndSave = `${btnVisible} && ${btnSave}`
 
         +btn-save(btnVisible, btnSave)
         .input-tip
-            button.select-toggle.form-control(id=javaTypeId ng-model=javaTypeModel data-placeholder='Java type' ng-class='{placeholder: !#{javaTypeModel}}' bs-select bs-options='item.value as item.label for item in {{supportedJavaTypes}}' ignite-on-enter=btnVisibleAndSave ignite-on-escape='tableReset(false)' tabindex='0')
+            button.select-toggle.form-control(id=javaTypeId ng-model=javaTypeModel data-placeholder='Java type' ng-class=`{placeholder: !${javaTypeModel}}` bs-select bs-options='item.value as item.label for item in {{supportedJavaTypes}}' ignite-on-enter=btnVisibleAndSave ignite-on-escape='tableReset(false)' tabindex='0')
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label(id='store-title') Domain model for cache store
         ignite-form-field-tooltip.tipLabel
             | Domain model properties for binding database with cache via POJO cache store#[br]
             | #[a(href="https://apacheignite.readme.io/docs/persistent-store" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
                     +text('Database schema:', model + '.databaseSchema', '"databaseSchema"', 'false', 'Input DB schema name', 'Schema name in database')
@@ -84,11 +84,11 @@ mixin table-db-field-edit(tbl, prefix, focusId, index)
                             | Collection of key fields descriptions for CacheJdbcPojoStore
                         ignite-form-group-add(ng-click='tableNewItem(keysTbl)')
                             | Add key field
-                        .group-content-empty(ng-show='!((#{keyFields} && #{keyFields}.length > 0) || tableNewItemActive(keysTbl))') Not defined
-                        .group-content(ng-show='(#{keyFields} && #{keyFields}.length > 0) || tableNewItemActive(keysTbl)')
+                        .group-content-empty(ng-show=`!((${keyFields} && ${keyFields}.length > 0) || tableNewItemActive(keysTbl))`) Not defined
+                        .group-content(ng-show=`(${keyFields} && ${keyFields}.length > 0) || tableNewItemActive(keysTbl)`)
                             table.links-edit(st-table=keyFields)
                                 tbody
-                                    tr(ng-repeat='item in #{keyFields} track by $index')
+                                    tr(ng-repeat=`item in ${keyFields} track by $index`)
                                         td
                                             div(ng-hide='tableEditing(keysTbl, $index)')
                                                 a.labelFormField(ng-click='tableStartEdit(backupItem, keysTbl, $index)') {{$index + 1}}) {{item.databaseFieldName}} / {{item.databaseFieldType}} / {{item.javaFieldName}} / {{item.javaFieldType}}
@@ -107,11 +107,11 @@ mixin table-db-field-edit(tbl, prefix, focusId, index)
                             | Collection of value fields descriptions for CacheJdbcPojoStore
                         ignite-form-group-add(ng-click='tableNewItem(valuesTbl)')
                             | Add value field
-                        .group-content-empty(ng-show='!((#{valueFields} && #{valueFields}.length > 0) || tableNewItemActive(valuesTbl))') Not defined
-                        .group-content(ng-show='(#{valueFields} && #{valueFields}.length > 0) || tableNewItemActive(valuesTbl)')
+                        .group-content-empty(ng-show=`!((${valueFields} && ${valueFields}.length > 0) || tableNewItemActive(valuesTbl))`) Not defined
+                        .group-content(ng-show=`(${valueFields} && ${valueFields}.length > 0) || tableNewItemActive(valuesTbl)`)
                             table.links-edit(st-table=valueFields)
                                 tbody
-                                    tr(ng-repeat='item in #{valueFields} track by $index')
+                                    tr(ng-repeat=`item in ${valueFields} track by $index`)
                                         td
                                             div(ng-hide='tableEditing(valuesTbl, $index)')
                                                 a.labelFormField(ng-click='tableStartEdit(backupItem, valuesTbl, $index)') {{$index + 1}}) {{item.databaseFieldName}} / {{item.databaseFieldType}} / {{item.javaFieldName}} / {{item.javaFieldType}}
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/dual.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/dual.pug
similarity index 75%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/dual.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/dual.pug
index f6ac89f223102..613070cbcdc44 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/dual.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/dual.pug
@@ -14,29 +14,29 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'dualMode'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Dual mode
         ignite-form-field-tooltip.tipLabel
             | IGFS supports dual-mode that allows it to work as either a standalone file system in Hadoop cluster, or work in tandem with HDFS, providing a primary caching layer for the secondary HDFS#[br]
             | As a caching layer it provides highly configurable read-through and write-through behaviour
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Maximum pending puts size:', model + '.dualModeMaxPendingPutsSize', '"dualModeMaxPendingPutsSize"', 'true', '0', 'Number.MIN_SAFE_INTEGER',
+                    +number('Maximum pending puts size:', `${model}.dualModeMaxPendingPutsSize`, '"dualModeMaxPendingPutsSize"', 'true', '0', 'Number.MIN_SAFE_INTEGER',
                         'Maximum amount of pending data read from the secondary file system and waiting to be written to data cache<br/>\
                         Zero or negative value stands for unlimited size')
                 .settings-row
-                    +java-class('Put executor service:', model + '.dualModePutExecutorService', '"dualModePutExecutorService"', 'true', 'false', 'DUAL mode put operation executor service')
+                    +java-class('Put executor service:', `${model}.dualModePutExecutorService`, '"dualModePutExecutorService"', 'true', 'false', 'DUAL mode put operation executor service')
                 .settings-row
-                    +checkbox('Put executor service shutdown', model + '.dualModePutExecutorServiceShutdown', '"dualModePutExecutorServiceShutdown"', 'DUAL mode put operation executor service shutdown flag')
+                    +checkbox('Put executor service shutdown', `${model}.dualModePutExecutorServiceShutdown`, '"dualModePutExecutorServiceShutdown"', 'DUAL mode put operation executor service shutdown flag')
             .col-sm-6
                 +preview-xml-java(model, 'igfsDualMode')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.pug
similarity index 70%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.pug
index 16f3749103904..b1126976272b8 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.pug
@@ -14,30 +14,30 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'fragmentizer'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Fragmentizer
         ignite-form-field-tooltip.tipLabel
             | Fragmentizer settings
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var enabled = model + '.fragmentizerEnabled'
+                -var enabled = `${model}.fragmentizerEnabled`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"fragmentizerEnabled"', 'Fragmentizer enabled flag')
                 .settings-row
-                    +number('Concurrent files:', model + '.fragmentizerConcurrentFiles', '"fragmentizerConcurrentFiles"', enabled, '0', '0', 'Number of files to process concurrently by fragmentizer')
+                    +number('Concurrent files:', `${model}.fragmentizerConcurrentFiles`, '"fragmentizerConcurrentFiles"', enabled, '0', '0', 'Number of files to process concurrently by fragmentizer')
                 .settings-row
-                    +number('Throttling block length:', model + '.fragmentizerThrottlingBlockLength', '"fragmentizerThrottlingBlockLength"', enabled, '16777216', '1', 'Length of file chunk to transmit before throttling is delayed')
+                    +number('Throttling block length:', `${model}.fragmentizerThrottlingBlockLength`, '"fragmentizerThrottlingBlockLength"', enabled, '16777216', '1', 'Length of file chunk to transmit before throttling is delayed')
                 .settings-row
-                    +number('Throttling delay:', model + '.fragmentizerThrottlingDelay', '"fragmentizerThrottlingDelay"', enabled, '200', '0', 'Delay in milliseconds for which fragmentizer is paused')
+                    +number('Throttling delay:', `${model}.fragmentizerThrottlingDelay`, '"fragmentizerThrottlingDelay"', enabled, '200', '0', 'Delay in milliseconds for which fragmentizer is paused')
             .col-sm-6
                 +preview-xml-java(model, 'igfsFragmentizer')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/general.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/general.pug
similarity index 89%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/general.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/general.pug
index 62cda770ceddc..e002d364675a9 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/general.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/general.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'general'
 -var model = 'backupItem'
@@ -31,11 +31,11 @@ include /app/helpers/jade/mixins.jade
         .panel-body
             .col-sm-6
                 .settings-row
-                    +text('Name:', model + '.name', '"igfsName"', 'true', 'Input name', 'IGFS name')
+                    +text('Name:', `${model}.name`, '"igfsName"', 'true', 'Input name', 'IGFS name')
                 .settings-row
                     +clusters(model, 'Associate clusters with the current IGFS')
                 .settings-row
-                    +dropdown('IGFS mode:', model + '.defaultMode', '"defaultMode"', 'true', 'DUAL_ASYNC',
+                    +dropdown('IGFS mode:', `${model}.defaultMode`, '"defaultMode"', 'true', 'DUAL_ASYNC',
                     '[\
                         {value: "PRIMARY", label: "PRIMARY"},\
                         {value: "PROXY", label: "PROXY"},\
@@ -50,7 +50,7 @@ include /app/helpers/jade/mixins.jade
                         <li>DUAL_ASYNC - in this mode IGFS will cache files locally and also <b> asynchronously </b> write them through to secondary file system</li>\
                     </ul>')
                 .settings-row
-                    +number('Group size:', model + '.affinnityGroupSize', '"affinnityGroupSize"', 'true', '512', '1',
+                    +number('Group size:', `${model}.affinnityGroupSize`, '"affinnityGroupSize"', 'true', '512', '1',
                         'Size of the group in blocks<br/>\
                         Required for construction of affinity mapper in IGFS data cache')
             .col-sm-6
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.pug
similarity index 66%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.pug
index 83c5de6eae68a..7c8c056a0ea2c 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.pug
@@ -14,28 +14,28 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'ipc'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label IPC
         ignite-form-field-tooltip.tipLabel
             | IGFS Inter-process communication properties
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var ipcEndpointConfiguration = model + '.ipcEndpointConfiguration'
-                -var enabled = model + '.ipcEndpointEnabled'
+                -var ipcEndpointConfiguration = `${model}.ipcEndpointConfiguration`
+                -var enabled = `${model}.ipcEndpointEnabled`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"ipcEndpointEnabled"', 'IPC endpoint enabled flag')
                 .settings-row
-                    +dropdown('Type:', ipcEndpointConfiguration + '.type', '"ipcEndpointConfigurationType"', enabled, 'TCP',
+                    +dropdown('Type:', `${ipcEndpointConfiguration}.type`, '"ipcEndpointConfigurationType"', enabled, 'TCP',
                         '[\
                             {value: "SHMEM", label: "SHMEM"},\
                             {value: "TCP", label: "TCP"}\
@@ -46,15 +46,15 @@ include /app/helpers/jade/mixins.jade
                             <li>TCP - TCP endpoint</li>\
                         </ul>')
                 .settings-row
-                    +text-ip-address('Host:', ipcEndpointConfiguration + '.host', '"ipcEndpointConfigurationHost"', enabled, '127.0.0.1', 'Host endpoint is bound to')
+                    +text-ip-address('Host:', `${ipcEndpointConfiguration}.host`, '"ipcEndpointConfigurationHost"', enabled, '127.0.0.1', 'Host endpoint is bound to')
                 .settings-row
-                    +number-min-max('Port:', ipcEndpointConfiguration + '.port', '"ipcEndpointConfigurationPort"', enabled, '10500', '1', '65535', 'Port endpoint is bound to')
+                    +number-min-max('Port:', `${ipcEndpointConfiguration}.port`, '"ipcEndpointConfigurationPort"', enabled, '10500', '1', '65535', 'Port endpoint is bound to')
                 .settings-row
-                    +number('Memory size:', ipcEndpointConfiguration + '.memorySize', '"ipcEndpointConfigurationMemorySize"', enabled, '262144', '1', 'Shared memory size in bytes allocated for endpoint communication')
+                    +number('Memory size:', `${ipcEndpointConfiguration}.memorySize`, '"ipcEndpointConfigurationMemorySize"', enabled, '262144', '1', 'Shared memory size in bytes allocated for endpoint communication')
                 .settings-row
-                    +text-enabled('Token directory:', ipcEndpointConfiguration + '.tokenDirectoryPath', '"ipcEndpointConfigurationTokenDirectoryPath"', enabled, 'false', 'ipc/shmem', 'Directory where shared memory tokens are stored')
+                    +text-enabled('Token directory:', `${ipcEndpointConfiguration}.tokenDirectoryPath`, '"ipcEndpointConfigurationTokenDirectoryPath"', enabled, 'false', 'ipc/shmem', 'Directory where shared memory tokens are stored')
                 .settings-row
-                    +number('Thread count:', ipcEndpointConfiguration + '.threadCount', '"ipcEndpointConfigurationThreadCount"', enabled, 'availableProcessors', '1',
+                    +number('Thread count:', `${ipcEndpointConfiguration}.threadCount`, '"ipcEndpointConfigurationThreadCount"', enabled, 'availableProcessors', '1',
                         'Number of threads used by this endpoint to process incoming requests')
             .col-sm-6
                 +preview-xml-java(model, 'igfsIPC')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/misc.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/misc.pug
similarity index 66%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/misc.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/misc.pug
index dc0e9fc73991c..aa4b95740566f 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/misc.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/misc.pug
@@ -14,68 +14,68 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'misc'
 -var model = 'backupItem'
 -var pathModesForm = 'miscPathModes'
--var pathModes = model + '.pathModes'
+-var pathModes = `${model}.pathModes`
 
 //- LEGACY mixin for LEGACY IGFS path modes table.
 mixin table-igfs-path-mode-edit(prefix, focusId, index)
-    -var keyModel = 'tblPathModes.' + prefix + 'Key'
-    -var valModel = 'tblPathModes.' + prefix + 'Value'
+    -var keyModel = `tblPathModes.${prefix}Key`
+    -var valModel = `tblPathModes.${prefix}Value`
 
-    -var keyFocusId = prefix + 'Key' + focusId
-    -var valFocusId = prefix + 'Value' + focusId
+    -var keyFocusId = `${prefix}Key${focusId}`
+    -var valFocusId = `${prefix}Value${focusId}`
 
     .col-xs-8.col-sm-8.col-md-8
         .fieldSep /
         .input-tip
             input.form-control(id=keyFocusId ignite-on-enter-focus-move=valFocusId type='text' ng-model=keyModel placeholder='Path' ignite-on-escape='tableReset(false)')
     .col-xs-4.col-sm-4.col-md-4
-        -var arg = keyModel + ', ' + valModel
-        -var btnVisible = 'tablePairSaveVisible(tblPathModes, ' + index + ')'
-        -var btnSave = 'tablePairSave(tablePairValid, backupItem, tblPathModes, ' + index + ')'
-        -var btnVisibleAndSave = btnVisible + ' && ' + btnSave
+        -var arg = `${keyModel}, ${valModel}`
+        -var btnVisible = `tablePairSaveVisible(tblPathModes, ${index})`
+        -var btnSave = `tablePairSave(tablePairValid, backupItem, tblPathModes, ${index})`
+        -var btnVisibleAndSave = `${btnVisible} && ${btnSave}`
         +btn-save(btnVisible, btnSave)
         .input-tip
             button.select-toggle.form-control(id=valFocusId bs-select ng-model=valModel data-placeholder='Mode' bs-options='item.value as item.label for item in igfsModes' tabindex='0' ignite-on-enter=btnVisibleAndSave ignite-on-escape='tableReset(false)')
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label Miscellaneous
         ignite-form-field-tooltip.tipLabel
             | Various miscellaneous IGFS settings
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
                 .settings-row
-                    +number('Block size:', model + '.blockSize', '"blockSize"', 'true', '65536', '0', 'File data block size in bytes')
+                    +number('Block size:', `${model}.blockSize`, '"blockSize"', 'true', '65536', '0', 'File data block size in bytes')
                 .settings-row
-                    +number('Stream buffer size:', model + '.streamBufferSize', '"streamBufferSize"', 'true', '65536', '0', 'Read/write buffer size for IGFS stream operations in bytes')
+                    +number('Stream buffer size:', `${model}.streamBufferSize`, '"streamBufferSize"', 'true', '65536', '0', 'Read/write buffer size for IGFS stream operations in bytes')
                 .settings-row
-                    +number('Maximum space size:', model + '.maxSpaceSize', '"maxSpaceSize"', 'true', '0', '0', 'Maximum space available for data cache to store file system entries')
+                    +number('Maximum space size:', `${model}.maxSpaceSize`, '"maxSpaceSize"', 'true', '0', '0', 'Maximum space available for data cache to store file system entries')
                 .settings-row
-                    +number('Maximum task range length:', model + '.maximumTaskRangeLength', '"maximumTaskRangeLength"', 'true', '0', '0', 'Maximum default range size of a file being split during IGFS task execution')
+                    +number('Maximum task range length:', `${model}.maximumTaskRangeLength`, '"maximumTaskRangeLength"', 'true', '0', '0', 'Maximum default range size of a file being split during IGFS task execution')
                 .settings-row
-                    +number-min-max('Management port:', model + '.managementPort', '"managementPort"', 'true', '11400', '0', '65535', 'Port number for management endpoint')
+                    +number-min-max('Management port:', `${model}.managementPort`, '"managementPort"', 'true', '11400', '0', '65535', 'Port number for management endpoint')
                 .settings-row
-                    +number('Per node batch size:', model + '.perNodeBatchSize', '"perNodeBatchSize"', 'true', '100', '0', 'Number of file blocks collected on local node before sending batch to remote node')
+                    +number('Per node batch size:', `${model}.perNodeBatchSize`, '"perNodeBatchSize"', 'true', '100', '0', 'Number of file blocks collected on local node before sending batch to remote node')
                 .settings-row
-                    +number('Per node parallel batch count:', model + '.perNodeParallelBatchCount', '"perNodeParallelBatchCount"', 'true', '8', '0', 'Number of file block batches that can be concurrently sent to remote node')
+                    +number('Per node parallel batch count:', `${model}.perNodeParallelBatchCount`, '"perNodeParallelBatchCount"', 'true', '8', '0', 'Number of file block batches that can be concurrently sent to remote node')
                 .settings-row
-                    +number('Prefetch blocks:', model + '.prefetchBlocks', '"prefetchBlocks"', 'true', '0', '0', 'Number of pre-fetched blocks if specific file chunk is requested')
+                    +number('Prefetch blocks:', `${model}.prefetchBlocks`, '"prefetchBlocks"', 'true', '0', '0', 'Number of pre-fetched blocks if specific file chunk is requested')
                 .settings-row
-                    +number('Sequential reads before prefetch:', model + '.sequentialReadsBeforePrefetch', '"sequentialReadsBeforePrefetch"', 'true', '0', '0', 'Amount of sequential block reads before prefetch is triggered')
+                    +number('Sequential reads before prefetch:', `${model}.sequentialReadsBeforePrefetch`, '"sequentialReadsBeforePrefetch"', 'true', '0', '0', 'Amount of sequential block reads before prefetch is triggered')
                 .settings-row
-                    +number('Trash purge timeout:', model + '.trashPurgeTimeout', '"trashPurgeTimeout"', 'true', '1000', '0', 'Maximum timeout awaiting for trash purging in case data cache oversize is detected')
+                    +number('Trash purge timeout:', `${model}.trashPurgeTimeout`, '"trashPurgeTimeout"', 'true', '1000', '0', 'Maximum timeout awaiting for trash purging in case data cache oversize is detected')
                 .settings-row
-                    +checkbox('Colocate metadata', model + '.colocateMetadata', '"colocateMetadata"', 'Whether to co-locate metadata on a single node')
+                    +checkbox('Colocate metadata', `${model}.colocateMetadata`, '"colocateMetadata"', 'Whether to co-locate metadata on a single node')
                 .settings-row
-                    +checkbox('Relaxed consistency', model + '.relaxedConsistency', '"relaxedConsistency"',
+                    +checkbox('Relaxed consistency', `${model}.relaxedConsistency`, '"relaxedConsistency"',
                         'If value of this flag is <b>true</b>, IGFS will skip expensive consistency checks<br/>\
                         It is recommended to set this flag to <b>false</b> if your application has conflicting\
                         operations, or you do not know how exactly users will use your system')
@@ -88,12 +88,12 @@ mixin table-igfs-path-mode-edit(prefix, focusId, index)
                         ignite-form-group-add(ng-click='tableNewItem(tblPathModes)')
                             | Add path mode
 
-                        .group-content-empty(ng-if='!((#{pathModes} && #{pathModes}.length > 0) || tableNewItemActive(tblPathModes))') Not defined
+                        .group-content-empty(ng-if=`!((${pathModes} && ${pathModes}.length > 0) || tableNewItemActive(tblPathModes))`) Not defined
 
-                        .group-content(ng-show='(#{pathModes} && #{pathModes}.length > 0) || tableNewItemActive(tblPathModes)')
+                        .group-content(ng-show=`(${pathModes} && ${pathModes}.length > 0) || tableNewItemActive(tblPathModes)`)
                             table.links-edit(id='pathModes' st-table=pathModes)
                                 tbody
-                                    tr(ng-repeat='item in #{pathModes} track by $index')
+                                    tr(ng-repeat=`item in ${pathModes} track by $index`)
                                         td.col-sm-12(ng-hide='tableEditing(tblPathModes, $index)')
                                             a.labelFormField(ng-click='tableStartEdit(backupItem, tblPathModes, $index)') {{item.path + " [" + item.mode + "]"}}
                                             +btn-remove('tableRemove(backupItem, tblPathModes, $index)', '"Remove path"')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.jade b/modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.pug
similarity index 69%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.jade
rename to modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.pug
index b605e7775a632..87e1f4fb8edb9 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.jade
+++ b/modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.pug
@@ -14,32 +14,32 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 -var form = 'secondaryFileSystem'
 -var model = 'backupItem'
 
 .panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
+    .panel-heading(bs-collapse-toggle='' ng-click=`ui.loadPanel('${form}')`)
         ignite-form-panel-chevron
         label(id="secondaryFileSystem-title") Secondary file system
         ignite-form-field-tooltip.tipLabel
             | Secondary file system is provided for pass-through, write-through, and read-through purposes#[br]
             | #[a(href="https://apacheignite-fs.readme.io/docs/secondary-file-system" target="_blank") More info]
         ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
+    .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+        .panel-body(ng-if=`ui.isPanelLoaded('${form}')`)
             .col-sm-6
-                -var enabled = model + '.secondaryFileSystemEnabled'
-                -var secondaryFileSystem = model + '.secondaryFileSystem'
+                -var enabled = `${model}.secondaryFileSystemEnabled`
+                -var secondaryFileSystem = `${model}.secondaryFileSystem`
 
                 .settings-row
                     +checkbox('Enabled', enabled, '"secondaryFileSystemEnabled"', 'Secondary file system enabled flag')
                 .settings-row
-                    +text-enabled('URI:', secondaryFileSystem + '.uri', '"hadoopURI"', enabled, 'false', 'hdfs://[namenodehost]:[port]/[path]', 'URI of file system')
+                    +text-enabled('URI:', `${secondaryFileSystem}.uri`, '"hadoopURI"', enabled, 'false', 'hdfs://[namenodehost]:[port]/[path]', 'URI of file system')
                 .settings-row
-                    +text-enabled('Config path:', secondaryFileSystem + '.cfgPath', '"cfgPath"', enabled, 'false', 'Path to additional config', 'Additional path to Hadoop configuration')
+                    +text-enabled('Config path:', `${secondaryFileSystem}.cfgPath`, '"cfgPath"', enabled, 'false', 'Path to additional config', 'Additional path to Hadoop configuration')
                 .settings-row
-                    +text-enabled('User name:', secondaryFileSystem + '.userName', '"userName"', enabled, 'false', 'Input user name', 'User name')
+                    +text-enabled('User name:', `${secondaryFileSystem}.userName`, '"userName"', enabled, 'false', 'Input user name', 'User name')
             .col-sm-6
                 +preview-xml-java(model, 'igfsSecondFS')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
index 16d2faef04335..25203b30d1a27 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
@@ -18,6 +18,8 @@
 import _ from 'lodash';
 import saver from 'file-saver';
 
+import summaryProjectStructureTemplateUrl from 'views/configuration/summary-project-structure.tpl.pug';
+
 const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
 
 export default [
@@ -25,6 +27,9 @@ export default [
     function($root, $scope, $http, LegacyUtils, Messages, Loading, $filter, Resource, JavaTypes, Version, generator, spring, java, docker, pom, propsGenerator, readme, FormUtils, SummaryZipper, ActivitiesData) {
         const ctrl = this;
 
+        // Define template urls.
+        ctrl.summaryProjectStructureTemplateUrl = summaryProjectStructureTemplateUrl;
+
         $scope.ui = {
             isSafari: !!(/constructor/i.test(window.HTMLElement) || window.safari),
             ready: false
diff --git a/modules/web-console/frontend/app/modules/states/errors.state.js b/modules/web-console/frontend/app/modules/states/errors.state.js
index 2bdb80a432461..e219132beb34e 100644
--- a/modules/web-console/frontend/app/modules/states/errors.state.js
+++ b/modules/web-console/frontend/app/modules/states/errors.state.js
@@ -16,8 +16,8 @@
  */
 
 import angular from 'angular';
-import templateNotFoundPage from '../../../views/404.jade';
-import templateNotAuthorizedPage from '../../../views/403.jade';
+import templateNotFoundPage from 'views/404.pug';
+import templateNotAuthorizedPage from 'views/403.pug';
 
 angular
     .module('ignite-console.states.errors', [
diff --git a/modules/web-console/frontend/app/modules/states/password.state.js b/modules/web-console/frontend/app/modules/states/password.state.js
index 48d01df60aa7f..587f83da9a1c5 100644
--- a/modules/web-console/frontend/app/modules/states/password.state.js
+++ b/modules/web-console/frontend/app/modules/states/password.state.js
@@ -17,6 +17,8 @@
 
 import angular from 'angular';
 
+import templateUrl from 'views/reset.tpl.pug';
+
 angular
 .module('ignite-console.states.password', [
     'ui.router'
@@ -31,14 +33,14 @@ angular
     })
     .state('password.reset', {
         url: '/reset?{token}',
-        templateUrl: '/reset.html',
+        templateUrl,
         metaTags: {
             title: 'Reset password'
         }
     })
     .state('password.send', {
         url: '/send',
-        templateUrl: '/reset.html',
+        templateUrl,
         metaTags: {
             title: 'Password Send'
         }
diff --git a/modules/web-console/frontend/app/modules/states/profile.state.js b/modules/web-console/frontend/app/modules/states/profile.state.js
index 9c31340a9569b..7c270ad17d36a 100644
--- a/modules/web-console/frontend/app/modules/states/profile.state.js
+++ b/modules/web-console/frontend/app/modules/states/profile.state.js
@@ -17,6 +17,8 @@
 
 import angular from 'angular';
 
+import templateUrl from 'views/settings/profile.tpl.pug';
+
 angular
 .module('ignite-console.states.profile', [
     'ui.router'
@@ -26,7 +28,7 @@ angular
     $stateProvider
     .state('settings.profile', {
         url: '/profile',
-        templateUrl: '/settings/profile.html',
+        templateUrl,
         onEnter: AclRoute.checkAccess('profile'),
         metaTags: {
             title: 'User profile'
diff --git a/modules/web-console/frontend/app/modules/states/signin.state.js b/modules/web-console/frontend/app/modules/states/signin.state.js
index 14ebc1baa6aab..2625aaa32997e 100644
--- a/modules/web-console/frontend/app/modules/states/signin.state.js
+++ b/modules/web-console/frontend/app/modules/states/signin.state.js
@@ -16,7 +16,7 @@
  */
 
 import angular from 'angular';
-import templateUrl from 'views/signin.jade';
+import templateUrl from 'views/signin.tpl.pug';
 
 angular
 .module('ignite-console.states.login', [
diff --git a/modules/web-console/frontend/app/services/Clone.service.js b/modules/web-console/frontend/app/services/Clone.service.js
index 52a4e4e8d1cb9..d079141364b55 100644
--- a/modules/web-console/frontend/app/services/Clone.service.js
+++ b/modules/web-console/frontend/app/services/Clone.service.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import templateUrl from 'views/templates/clone.tpl.pug';
+
 // Service for clone objects.
 export default ['IgniteClone', ['$rootScope', '$q', '$modal', ($root, $q, $modal) => {
     const scope = $root.$new();
@@ -36,7 +38,7 @@ export default ['IgniteClone', ['$rootScope', '$q', '$modal', ($root, $q, $modal
         return tmpName;
     }
 
-    const cloneModal = $modal({templateUrl: '/templates/clone.html', scope, placement: 'center', show: false});
+    const cloneModal = $modal({templateUrl, scope, placement: 'center', show: false});
 
     scope.ok = function(newName) {
         if (!_validator || _validator(newName)) {
diff --git a/modules/web-console/frontend/app/services/Confirm.service.js b/modules/web-console/frontend/app/services/Confirm.service.js
index 8208ea2a15454..0b5b9be87a6ad 100644
--- a/modules/web-console/frontend/app/services/Confirm.service.js
+++ b/modules/web-console/frontend/app/services/Confirm.service.js
@@ -15,11 +15,13 @@
  * limitations under the License.
  */
 
+import templateUrl from 'views/templates/confirm.tpl.pug';
+
 // Confirm popup service.
 export default ['IgniteConfirm', ['$rootScope', '$q', '$modal', '$animate', ($root, $q, $modal, $animate) => {
     const scope = $root.$new();
 
-    const modal = $modal({templateUrl: '/templates/confirm.html', scope, placement: 'center', show: false, backdrop: true});
+    const modal = $modal({templateUrl, scope, placement: 'center', show: false, backdrop: true});
 
     const _hide = () => {
         $animate.enabled(modal.$element, false);
diff --git a/modules/web-console/frontend/app/services/ConfirmBatch.service.js b/modules/web-console/frontend/app/services/ConfirmBatch.service.js
index ef663353539aa..2fd55b709d8c4 100644
--- a/modules/web-console/frontend/app/services/ConfirmBatch.service.js
+++ b/modules/web-console/frontend/app/services/ConfirmBatch.service.js
@@ -15,12 +15,14 @@
  * limitations under the License.
  */
 
+import templateUrl from 'views/templates/batch-confirm.tpl.pug';
+
 // Service for confirm or skip several steps.
 export default ['IgniteConfirmBatch', ['$rootScope', '$q', '$modal', ($root, $q, $modal) => {
     const scope = $root.$new();
 
     scope.confirmModal = $modal({
-        templateUrl: '/templates/batch-confirm.html',
+        templateUrl,
         scope,
         placement: 'center',
         show: false,
diff --git a/modules/web-console/frontend/controllers/caches-controller.js b/modules/web-console/frontend/controllers/caches-controller.js
index e7521b55ad403..b50fde3188b1a 100644
--- a/modules/web-console/frontend/controllers/caches-controller.js
+++ b/modules/web-console/frontend/controllers/caches-controller.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import infoMessageTemplateUrl from 'views/templates/message.tpl.pug';
+
 // Controller for Caches screen.
 export default ['cachesController', [
     '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'IgniteLegacyTable',
@@ -542,7 +544,7 @@ export default ['cachesController', [
                         ];
 
                         // Show a basic modal from a controller
-                        $modal({scope, template: '/templates/message.html', placement: 'center', show: true});
+                        $modal({scope, templateUrl: infoMessageTemplateUrl, placement: 'center', show: true});
                     }
 
                     save(item);
diff --git a/modules/web-console/frontend/controllers/domains-controller.js b/modules/web-console/frontend/controllers/domains-controller.js
index bfffe92f62747..5c9e511a139ca 100644
--- a/modules/web-console/frontend/controllers/domains-controller.js
+++ b/modules/web-console/frontend/controllers/domains-controller.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import templateUrl from 'views/configuration/domains-import.tpl.pug';
+
 // Controller for Domain model screen.
 export default ['domainsController', [
     '$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteAgentMonitor', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes', 'IgniteActivitiesData',
@@ -409,7 +411,7 @@ export default ['domainsController', [
         $scope.$watch('importDomain.displayedTables', $scope.selectTable);
 
         // Pre-fetch modal dialogs.
-        const importDomainModal = $modal({scope: $scope, templateUrl: '/configuration/domains-import.html', show: false});
+        const importDomainModal = $modal({scope: $scope, templateUrl, show: false});
 
         const hideImportDomain = importDomainModal.hide;
 
diff --git a/modules/web-console/frontend/gulpfile.babel.js/paths.js b/modules/web-console/frontend/gulpfile.babel.js/paths.js
index 6ebfbad42ba84..627b2f9599058 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/paths.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/paths.js
@@ -24,33 +24,15 @@ const destDir = path.resolve('build');
 const igniteModulesDir = process.env.IGNITE_MODULES ? path.join(path.normalize(process.env.IGNITE_MODULES), 'frontend') : './ignite_modules';
 const igniteModulesTemp = path.resolve('ignite_modules_temp');
 
-const jadeViewsPaths = [
-    './views/**/*.jade',
-    '!./views/configuration/*.jade'
-];
-
-const jadeAppModulePaths = [
-    './app/modules/states/configuration/**/*.jade',
-    './app/modules/sql/*.jade',
-    './views/**/*.jade',
-    '!./views/*.jade',
-    '!./views/includes/*.jade',
-    '!./views/settings/*.jade',
-    '!./views/sql/*.jade',
-    '!./views/templates/*.jade'
-];
-
-const jadeModulePaths = [
-    igniteModulesDir + '/**/view/**/*.jade'
-];
-
 const appModulePaths = [
     igniteModulesDir + '/index.js',
     igniteModulesDir + '/**/main.js',
     igniteModulesDir + '/**/module.js',
     igniteModulesDir + '/**/app/modules/*.js',
     igniteModulesDir + '/**/app/modules/**/*.js',
-    igniteModulesDir + '/**/app/modules/**/*.jade',
+    igniteModulesDir + '/**/app/modules/**/*.pug',
+    igniteModulesDir + '/**/*.pug',
+    igniteModulesDir + '/**/*.tpl.pug',
     igniteModulesDir + '/**/app/**/*.css',
     igniteModulesDir + '/**/app/**/*.scss',
     igniteModulesDir + '/**/app/data/*.json'
@@ -73,10 +55,6 @@ export {
     igniteModulesDir,
     igniteModulesTemp,
 
-    jadeViewsPaths,
-    jadeAppModulePaths,
-    jadeModulePaths,
-
     resourcePaths,
     resourceModulePaths,
     appModulePaths
diff --git a/modules/web-console/frontend/gulpfile.babel.js/tasks/build.js b/modules/web-console/frontend/gulpfile.babel.js/tasks/build.js
index 7d7401b7cfb4a..149b07680c8a9 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/tasks/build.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/tasks/build.js
@@ -18,4 +18,4 @@
 import gulp from 'gulp';
 import sequence from 'gulp-sequence';
 
-gulp.task('build', (cb) => sequence(['clean', 'clean:ignite-modules-temp'], 'ignite:modules', ['copy:resource', 'jade'], 'bundle', cb));
+gulp.task('build', (cb) => sequence(['clean', 'clean:ignite-modules-temp'], 'ignite:modules', ['copy:resource'], 'bundle', cb));
diff --git a/modules/web-console/frontend/gulpfile.babel.js/tasks/bundle.js b/modules/web-console/frontend/gulpfile.babel.js/tasks/bundle.js
index d3e8dca576cd5..ac3ac76be89d1 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/tasks/bundle.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/tasks/bundle.js
@@ -21,10 +21,14 @@ import webpackConfig from '../webpack';
 import WebpackDevServer from 'webpack-dev-server';
 
 gulp.task('bundle', (cb) => {
+    const devServerConfig = webpackConfig.devServer;
+
+    delete webpackConfig.devServer;
+
     if (process.env.NODE_ENV === 'development') {
         // Important! Call webpack and WebpackDevServer must be inline.
-        new WebpackDevServer(webpack(webpackConfig), webpackConfig.devServer)
-            .listen(webpackConfig.devServer.port, 'localhost', cb);
+        new WebpackDevServer(webpack(webpackConfig), devServerConfig)
+            .listen(devServerConfig.port, 'localhost', cb);
     }
     else
         webpack(webpackConfig, cb);
diff --git a/modules/web-console/frontend/gulpfile.babel.js/tasks/watch.js b/modules/web-console/frontend/gulpfile.babel.js/tasks/watch.js
index dfaa1a3a88c35..9a02b889a6b41 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/tasks/watch.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/tasks/watch.js
@@ -18,15 +18,12 @@
 import gulp from 'gulp';
 import sequence from 'gulp-sequence';
 
-import { jadeViewsPaths, jadeAppModulePaths, jadeModulePaths, resourcePaths, resourceModulePaths, appModulePaths } from '../paths';
+import { resourcePaths, resourceModulePaths, appModulePaths } from '../paths';
 
 gulp.task('watch:ignite-modules', (cb) => sequence('clean:ignite-modules-temp', 'ignite:modules', cb));
 
 // Build + watch task.
 gulp.task('watch', ['build'], () => {
-    gulp.watch(jadeViewsPaths, ['jade:views']);
-    gulp.watch(jadeAppModulePaths.concat('./app/helpers/**/*.jade'), ['jade:app']);
-    gulp.watch(jadeModulePaths.concat('./app/helpers/**/*.jade'), ['jade:ignite_modules']);
     gulp.watch(resourcePaths, ['copy:resource:app']);
     gulp.watch(resourceModulePaths, ['copy:resource:ignite_modules']);
     gulp.watch(appModulePaths, ['watch:ignite-modules']);
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js
index 7360ac4c0da3c..3d9616f788d1d 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/common.js
@@ -18,21 +18,22 @@
 import path from 'path';
 import fs from 'fs';
 import webpack from 'webpack';
-import autoprefixer from 'autoprefixer-core';
-import jade from 'jade';
+
 import ProgressBarPlugin from 'progress-bar-webpack-plugin';
 import eslintFormatter from 'eslint-friendly-formatter';
 
-import ExtractTextPlugin from 'extract-text-webpack-plugin';
 import HtmlWebpackPlugin from 'html-webpack-plugin';
 
+import ExtractTextPlugin from 'extract-text-webpack-plugin';
+
 import {srcDir, destDir, rootDir, igniteModulesDir} from '../paths';
 
+const viewsDir = path.resolve('views');
+const imagesDir = path.resolve('public/images');
+
 const NODE_ENV = process.env.NODE_ENV || 'production';
 const development = NODE_ENV === 'development';
 const node_modules_path = path.resolve('node_modules');
-const cssLoader = 'css-loader?sourceMap!postcss-loader';
-const stylesLoader = cssLoader + '!sass-loader?outputStyle=expanded&sourceMap=true&sourceMapContents=true';
 
 let favicon = 'build/ignite_modules/favicon.ico';
 
@@ -42,150 +43,155 @@ try {
     favicon = 'build/favicon.ico';
 }
 
-export default () => {
-    const assetsLoader = 'file-loader';
-
-    return {
-        cache: true,
-        node: {
-            fs: 'empty'
-        },
-
-        // Entry points.
-        entry: {
-            polyfill: 'babel-polyfill',
-            app: path.join(srcDir, 'app.js'),
-            vendor: path.join(srcDir, 'vendor.js')
-        },
-
-        // Output system.
-        output: {
-            path: destDir,
-            filename: '[name].js'
-        },
-
-        // Resolves modules.
-        resolve: {
-            extensions: [
-                '',
-                '.js'
-            ],
-            root: [rootDir],
-            modulesDirectories: [
-                node_modules_path,
-                './'
-            ]
-        },
-        jade: {
-            basedir: rootDir,
-            locals: {}
-        },
-        // Modules resolvers.
-        /* global require */
-        module: {
-            noParse: [],
-            preLoaders: [
-                {
-                    test: /\.js$/,
-                    exclude: [node_modules_path],
-                    loader: 'eslint-loader'
-                }
-            ],
-            loaders: [
-                {
-                    test: /\.json$/,
-                    loader: 'json-loader'
-                },
-                {
-                    test: /\.jade$/,
-                    loaders: [
-                        `ngtemplate-loader?relativeTo=${rootDir}`,
-                        'html-loader?attrs[]=img:src&attrs[]=img:data-src',
-                        'jade-html-loader'
-                    ]
-                },
-                {
-                    test: /\.js$/,
-                    exclude: [node_modules_path],
-                    loader: 'babel-loader',
-                    query: {
+export default {
+    cache: true,
+    node: {
+        fs: 'empty'
+    },
+    // Entry points.
+    entry: {
+        polyfill: 'babel-polyfill',
+        vendor: path.join(srcDir, 'vendor.js'),
+        app: path.join(srcDir, 'app.js')
+    },
+
+    // Output system.
+    output: {
+        path: destDir,
+        filename: '[name].js'
+    },
+
+    // Resolves modules.
+    resolve: {
+        extensions: [
+            '.js'
+        ],
+        modules: [
+            srcDir,
+            rootDir,
+            node_modules_path
+        ],
+        // A list of module source folders.
+        alias: {
+            app: srcDir,
+            views: viewsDir,
+            images: imagesDir
+        }
+    },
+
+    // Resolve loader use postfix.
+    resolveLoader: {
+        moduleExtensions: ['-loader']
+    },
+
+    module: {
+        rules: [
+            {
+                test: /\.json$/,
+                loader: 'json'
+            },
+
+            // Exclude tpl.pug files to import in bundle.
+            {
+                test: /^(?:(?!tpl\.pug$).)*\.pug$/, // TODO: check this regexp for correct.
+                loader: `pug-html?basedir=${rootDir}`
+            },
+
+            // Render .tpl.pug files to assets folder.
+            {
+                test: /\.tpl\.pug$/,
+                use: [
+                    'file?exports=false&name=assets/templates/[name].[hash].html',
+                    `pug-html?exports=false&basedir=${rootDir}`
+                ]
+            },
+            {
+                test: /\.js$/,
+                enforce: 'pre',
+                exclude: [node_modules_path],
+                use: [{
+                    loader: 'eslint',
+                    options: {
+                        failOnWarning: false,
+                        failOnError: false,
+                        formatter: eslintFormatter
+                    }
+                }]
+            },
+            {
+                test: /\.js$/,
+                exclude: [node_modules_path],
+                use: [{
+                    loader: 'babel',
+                    options: {
                         cacheDirectory: true,
                         plugins: [
                             'transform-runtime',
                             'add-module-exports'
                         ],
                         presets: ['angular']
-
                     }
-                },
-                {
-                    test: /\.css$/,
-                    loader: development ? `style-loader!${cssLoader}` : ExtractTextPlugin.extract('style-loader', cssLoader)
-                },
-                {
-                    test: /\.(scss|sass)$/,
-                    loader: development ? `style-loader!${stylesLoader}` : ExtractTextPlugin.extract('style-loader', stylesLoader)
-                },
-                {
-                    test: /\.(ttf|eot|svg|woff(2)?)(\?v=[\d.]+)?(\?[a-z0-9#-]+)?$/,
-                    loaders: [`${assetsLoader}?name=assets/fonts/[name].[ext]`]
-                },
-                {
-                    test: /\.(jpe?g|png|gif)$/i,
-                    loaders: [`${assetsLoader}?name=assets/images/[name]_[hash].[ext]`]
-                },
-                {
-                    test: require.resolve('jquery'),
-                    loaders: [
-                        'expose-loader?$',
-                        'expose-loader?jQuery'
-                    ]
-                },
-                {
-                    test: require.resolve('nvd3'),
-                    loaders: [
-                        'expose-loader?nv'
-                    ]
-                }
-            ]
-        },
-
-        // Postcss configuration.
-        postcss: [autoprefixer({browsers: ['last 2 versions']})],
-
-        // ESLint loader configuration.
-        eslint: {
-            failOnWarning: false,
-            failOnError: false,
-            formatter: eslintFormatter
-        },
-
-        // Load plugins.
-        plugins: [
-            new webpack.ProvidePlugin({
-                $: 'jquery',
-                jQuery: 'jquery',
-                _: 'lodash',
-                nv: 'nvd3'
-            }),
-            new webpack.DefinePlugin({NODE_ENV: JSON.stringify(NODE_ENV)}),
-            // new webpack.NoErrorsPlugin(),
-            new webpack.optimize.DedupePlugin(),
-            new webpack.optimize.CommonsChunkPlugin({
-                name: 'common',
-                chunks: ['vendor', 'app']
-            }),
-            new webpack.optimize.AggressiveMergingPlugin({moveToParents: true}),
-            new webpack.optimize.OccurenceOrderPlugin(),
-            new ExtractTextPlugin('assets/css/[name]' + (development ? '' : '.[chunkhash]') + '.css', {allChunks: true}),
-            new HtmlWebpackPlugin({
-                filename: 'index.html',
-                templateContent: () => {
-                    return jade.renderFile(path.join(rootDir, 'views', 'index.jade'));
-                },
-                favicon
-            }),
-            new ProgressBarPlugin()
+                }]
+            },
+            {
+                test: /\.css$/,
+                use: development ? ['style', 'css'] : ExtractTextPlugin.extract({
+                    fallback: 'style',
+                    use: ['css']
+                })
+            },
+            {
+                test: /\.scss$/,
+                use: development ? ['style', 'css', 'sass'] : ExtractTextPlugin.extract({
+                    fallback: 'style-loader',
+                    use: ['css', 'sass']
+                })
+            },
+            {
+                test: /\.(ttf|eot|svg|woff(2)?)(\?v=[\d.]+)?(\?[a-z0-9#-]+)?$/,
+                loader: 'file?name=assets/fonts/[name].[ext]'
+            },
+            {
+                test: /\.(jpe?g|png|gif)$/i,
+                loader: 'file?name=assets/images/[name]_[hash].[ext]'
+            },
+            {
+                test: require.resolve('jquery'),
+                use: [
+                    'expose-loader?$',
+                    'expose-loader?jQuery'
+                ]
+            },
+            {
+                test: require.resolve('nvd3'),
+                use: ['expose-loader?nv']
+            }
         ]
-    };
+    },
+
+    // Load plugins.
+    plugins: [
+        new webpack.LoaderOptionsPlugin({
+            options: {
+                pug: {
+                    basedir: rootDir
+                }
+            }
+        }),
+        new webpack.ProvidePlugin({
+            $: 'jquery',
+            jQuery: 'jquery',
+            _: 'lodash',
+            nv: 'nvd3'
+        }),
+        new webpack.DefinePlugin({NODE_ENV: JSON.stringify(NODE_ENV)}),
+        new webpack.optimize.CommonsChunkPlugin({name: 'vendor'}),
+        new webpack.optimize.AggressiveMergingPlugin({moveToParents: true}),
+        new HtmlWebpackPlugin({
+            template: './views/index.pug',
+            favicon
+        }),
+        new ExtractTextPlugin({filename: 'assets/css/[name].css', allChunks: true}),
+        new ProgressBarPlugin()
+    ]
 };
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js
index 34e1f6ae858b0..c837c43d828d5 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/development.js
@@ -15,65 +15,47 @@
  * limitations under the License.
  */
 
-import path from 'path';
-import webpack from 'webpack';
-
-import {destDir, rootDir, srcDir} from '../../paths';
+import {destDir} from '../../paths';
 
 const backendPort = 3000;
 const devServerPort = 9000;
 
-export default () => {
-    const plugins = [
-        new webpack.HotModuleReplacementPlugin()
-    ];
-
-    return {
-        entry: {
-            app: [path.join(srcDir, 'app.js'), 'webpack/hot/only-dev-server']
-        },
-        output: {
-            publicPath: `http://localhost:${devServerPort}/`
-        },
-        context: rootDir,
-        debug: true,
-        devtool: 'source-map',
-        watch: true,
-        devServer: {
-            compress: true,
-            historyApiFallback: true,
-            contentBase: destDir,
-            hot: true,
-            inline: true,
-            proxy: {
-                '/socket.io': {
-                    target: `http://localhost:${backendPort}`,
-                    changeOrigin: true,
-                    ws: true
-                },
-                '/agents': {
-                    target: `http://localhost:${backendPort}`,
-                    changeOrigin: true,
-                    ws: true
-                },
-                '/api/v1/*': {
-                    target: `http://localhost:${backendPort}`,
-                    changeOrigin: true,
-                    pathRewrite: {
-                        '^/api/v1': ''
-                    }
-                }
+export default {
+    devtool: 'source-map',
+    watch: true,
+    devServer: {
+        compress: true,
+        historyApiFallback: true,
+        contentBase: destDir,
+        // hot: true,
+        inline: true,
+        proxy: {
+            '/socket.io': {
+                target: `http://localhost:${backendPort}`,
+                changeOrigin: true,
+                ws: true
             },
-            watchOptions: {
-                aggregateTimeout: 1000,
-                poll: 2000
+            '/agents': {
+                target: `http://localhost:${backendPort}`,
+                changeOrigin: true,
+                ws: true
             },
-            stats: {
-                colors: true,
-                chunks: false
-            },
-            port: devServerPort
+            '/api/v1/*': {
+                target: `http://localhost:${backendPort}`,
+                changeOrigin: true,
+                pathRewrite: {
+                    '^/api/v1': ''
+                }
+            }
+        },
+        watchOptions: {
+            aggregateTimeout: 1000,
+            poll: 2000
+        },
+        stats: {
+            colors: true,
+            chunks: false
         },
-        plugins
-    };
+        port: devServerPort
+    }
 };
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js
index 1194568f79217..cbbe9ca3a9f2e 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/production.js
@@ -15,30 +15,27 @@
  * limitations under the License.
  */
 
+
 import webpack from 'webpack';
 
-import {destDir, rootDir} from '../../paths';
+import {destDir} from '../../paths';
 
-export default () => {
-    const plugins = [
-        new webpack.optimize.UglifyJsPlugin({
-            path: destDir,
-            minimize: true,
-            warnings: false,
-            sourceMap: false,
-            mangle: true
-        })
-    ];
+const plugins = [
+    new webpack.optimize.UglifyJsPlugin({
+        path: destDir,
+        minimize: true,
+        warnings: false,
+        sourceMap: false,
+        mangle: true
+    })
+];
 
-    return {
-        context: rootDir,
-        bail: true, // Cancel build on error.
-        debug: false,
-        devtool: 'cheap-source-map',
-        output: {
-            publicPath: '/',
-            filename: '[name].[chunkhash].js'
-        },
-        plugins
-    };
+export default {
+    bail: true, // Cancel build on error.
+    devtool: 'cheap-source-map',
+    output: {
+        publicPath: '/',
+        filename: '[name].[chunkhash].js'
+    },
+    plugins
 };
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/index.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/index.js
index 9b344dd6e1cab..3caf06c9c4728 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/index.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/index.js
@@ -19,6 +19,8 @@ import _ from 'lodash';
 import commonConfig from './common';
 import devConfig from './environments/development';
 import prodConfig from './environments/production';
+
+// TODO check test config
 import testConfig from './environments/test';
 
 const env = process.env.NODE_ENV || 'production';
@@ -31,4 +33,4 @@ const configs = {
 };
 
 // Load config file by environment
-export default _.merge(commonConfig(), configs[env]());
+export default _.merge(commonConfig, configs[env]);
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 2d126550f16b4..b11f690d66e9e 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -5,7 +5,7 @@
   "private": true,
   "scripts": {
     "dev": "cross-env NODE_ENV=development gulp watch",
-    "build": "cross-env NODE_ENV=production gulp build --no-ll",
+    "build": "cross-env NODE_ENV=production gulp build",
     "test": "cross-env NODE_ENV=test karma start ./test/karma.conf.js",
     "eslint": "eslint --format node_modules/eslint-friendly-formatter gulpfile.babel.js/ app/ controllers/ ignite_modules/ ignite_modules_temp/ -- --eff-by-issue"
   },
@@ -50,7 +50,7 @@
     "angular-ui-router": "0.4.2",
     "bootstrap-sass": "3.3.7",
     "brace": "0.8.0",
-    "es6-promise": "3.3.1",
+    "es6-promise": "4.0.5",
     "file-saver": "1.3.3",
     "font-awesome": "4.7.0",
     "glob": "7.1.1",
@@ -60,12 +60,12 @@
     "nvd3": "1.8.4",
     "raleway-webfont": "3.0.1",
     "roboto-font": "0.1.0",
-    "socket.io-client": "1.7.2",
+    "socket.io-client": "1.7.3",
     "ui-router-metatags": "1.0.3"
   },
   "devDependencies": {
     "assets-webpack-plugin": "3.5.1",
-    "autoprefixer-core": "6.0.1",
+    "autoprefixer": "6.7.5",
     "babel-core": "6.20.0",
     "babel-eslint": "7.0.0",
     "babel-loader": "6.2.10",
@@ -77,52 +77,50 @@
     "babel-preset-es2015": "6.18.0",
     "babel-runtime": "6.20.0",
     "chai": "3.5.0",
-    "cross-env": "1.0.8",
-    "css-loader": "0.26.1",
-    "eslint": "3.12.2",
+    "cross-env": "3.1.4",
+    "css-loader": "0.26.2",
+    "eslint": "3.16.1",
     "eslint-friendly-formatter": "2.0.7",
-    "eslint-loader": "1.6.1",
-    "expose-loader": "0.7.1",
-    "extract-text-webpack-plugin": "1.0.1",
-    "file-loader": "0.9.0",
+    "eslint-loader": "1.6.3",
+    "expose-loader": "0.7.3",
+    "extract-text-webpack-plugin": "2.0.0",
+    "file-loader": "0.10.1",
     "gulp": "3.9.1",
     "gulp-eslint": "3.0.1",
-    "gulp-inject": "4.1.0",
-    "gulp-jade": "1.1.0",
-    "gulp-ll": "1.0.4",
+    "gulp-inject": "4.2.0",
     "gulp-rimraf": "0.2.1",
     "gulp-sequence": "0.4.6",
     "gulp-util": "3.0.8",
-    "html-loader": "0.4.4",
-    "html-webpack-plugin": "2.24.1",
-    "jade": "1.11.0",
-    "jade-html-loader": "git://github.com/courcelan/jade-html-loader",
+    "html-loader": "0.4.5",
+    "html-webpack-plugin": "2.28.0",
     "jasmine-core": "2.5.2",
     "json-loader": "0.5.4",
-    "karma": "0.13.22",
+    "karma": "1.5.0",
     "karma-babel-preprocessor": "6.0.1",
     "karma-jasmine": "1.1.0",
     "karma-mocha": "1.3.0",
     "karma-mocha-reporter": "2.2.2",
     "karma-phantomjs-launcher": "1.0.2",
     "karma-teamcity-reporter": "1.0.0",
-    "karma-webpack": "1.8.1",
-    "mocha": "2.5.3",
+    "karma-webpack": "2.0.2",
+    "mocha": "3.2.0",
     "mocha-teamcity-reporter": "1.1.1",
-    "morgan": "1.7.0",
     "ngtemplate-loader": "1.3.1",
-    "node-sass": "3.13.1",
+    "node-sass": "4.5.0",
     "phantomjs-prebuilt": "2.1.14",
-    "postcss-loader": "0.9.1",
+    "postcss-loader": "1.3.2",
     "progress-bar-webpack-plugin": "1.9.3",
+    "pug-html-loader": "1.1.0",
+    "pug-loader": "2.3.0",
     "require-dir": "0.3.1",
-    "resolve-url-loader": "1.6.1",
-    "sass-loader": "3.2.2",
-    "style-loader": "0.13.1",
+    "resolve-url-loader": "2.0.0",
+    "sass-loader": "6.0.2",
+    "style-loader": "0.13.2",
     "url": "0.11.0",
-    "url-loader": "0.5.7",
-    "webpack": "1.14.0",
-    "webpack-dev-server": "1.16.3",
-    "worker-loader": "0.7.1"
+    "url-loader": "0.5.8",
+    "webpack": "2.2.1",
+    "webpack-dev-server": "2.4.1",
+    "webpack-stream": "3.2.0",
+    "worker-loader": "0.8.0"
   }
 }
diff --git a/modules/web-console/frontend/test/e2e/exampe.test.js b/modules/web-console/frontend/test/e2e/exampe.test.js
index c778c797e61b7..00788bbfbe8dd 100644
--- a/modules/web-console/frontend/test/e2e/exampe.test.js
+++ b/modules/web-console/frontend/test/e2e/exampe.test.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import { suite, test, setup } from 'mocha';
+
 suite('ExampleTestSuite', () => {
     setup(() => {
         // browser.get('http://localhost:9000/');
@@ -37,4 +39,4 @@ suite('ExampleTestSuite', () => {
 
         // element(by.model('ui.email')).sendKeys('jhon@doe.com');
     });
-});
\ No newline at end of file
+});
diff --git a/modules/web-console/frontend/test/karma.conf.babel.js b/modules/web-console/frontend/test/karma.conf.babel.js
index 76a0ba0a240a5..992c4d07a2990 100644
--- a/modules/web-console/frontend/test/karma.conf.babel.js
+++ b/modules/web-console/frontend/test/karma.conf.babel.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import webpackConfig from '../gulpfile.babel.js/webpack';
+import webpack from '../gulpfile.babel.js/webpack';
 import path from 'path';
 
 const basePath = path.resolve('./');
@@ -23,7 +23,7 @@ const basePath = path.resolve('./');
 export default (config) => {
     config.set({
         // Base path that will be used to resolve all patterns (eg. files, exclude).
-        basePath: basePath,
+        basePath,
 
         // Frameworks to use available frameworks: https://npmjs.org/browse/keyword/karma-adapter
         frameworks: ['mocha'],
@@ -46,7 +46,8 @@ export default (config) => {
         preprocessors: {
             'test/**/*.js': ['webpack']
         },
-        webpack: webpackConfig,
+
+        webpack,
 
         webpackMiddleware: {
             noInfo: true
diff --git a/modules/web-console/frontend/test/protractor.conf.js b/modules/web-console/frontend/test/protractor.conf.js
index 3386e66b9f115..4ee1c69078108 100644
--- a/modules/web-console/frontend/test/protractor.conf.js
+++ b/modules/web-console/frontend/test/protractor.conf.js
@@ -23,10 +23,10 @@
 // };
 
 exports.config = {
-  seleniumAddress: 'http://localhost:4444/wd/hub',
+    seleniumAddress: 'http://localhost:4444/wd/hub',
 
-  capabilities: {
-    'browserName': 'chrome'
+    capabilities: {
+        browserName: 'chrome'
     // 'browserName': 'phantomjs',
 
     // /*
@@ -40,11 +40,11 @@ exports.config = {
     //  * See https://github.com/detro/ghostdriver#faq
     //  */
     // 'phantomjs.ghostdriver.cli.args': ['--loglevel=DEBUG']
-  },
+    },
 
-  specs: ['test/e2e/*.js'],
+    specs: ['test/e2e/*.js'],
 
-  jasmineNodeOpts: {
-    showColors: true
-  }
+    jasmineNodeOpts: {
+        showColors: true
+    }
 };
diff --git a/modules/web-console/frontend/test/unit/JavaTransformer.test.js b/modules/web-console/frontend/test/unit/JavaTransformer.test.js
index 3f390003722da..7dacc1681e093 100644
--- a/modules/web-console/frontend/test/unit/JavaTransformer.test.js
+++ b/modules/web-console/frontend/test/unit/JavaTransformer.test.js
@@ -20,7 +20,7 @@ import JavaTypes from '../../app/services/JavaTypes.service.js';
 import generator from '../../app/modules/configuration/generator/ConfigurationGenerator';
 import transformer from '../../app/modules/configuration/generator/JavaTransformer.service';
 
-import { assert } from 'chai';
+import { suite, test } from 'mocha';
 
 suite.skip('JavaTransformerTestsSuite', () => {
     test('AtomicConfiguration', () => {
diff --git a/modules/web-console/frontend/test/unit/JavaTypes.test.js b/modules/web-console/frontend/test/unit/JavaTypes.test.js
index 49e78cc75efd4..e42d687e15eaf 100644
--- a/modules/web-console/frontend/test/unit/JavaTypes.test.js
+++ b/modules/web-console/frontend/test/unit/JavaTypes.test.js
@@ -21,113 +21,114 @@ import ClusterDflts from '../../app/modules/configuration/generator/defaults/Clu
 import CacheDflts from '../../app/modules/configuration/generator/defaults/Cache.service';
 import IgfsDflts from '../../app/modules/configuration/generator/defaults/IGFS.service';
 
-const INSTANCE = new JavaTypes(new ClusterDflts(), new CacheDflts(), new IgfsDflts());
+const instance = new JavaTypes(new ClusterDflts(), new CacheDflts(), new IgfsDflts());
 
+import { suite, test } from 'mocha';
 import { assert } from 'chai';
 
 suite('JavaTypesTestsSuite', () => {
     test('nonBuiltInClass', () => {
-        assert.equal(INSTANCE.nonBuiltInClass('BigDecimal'), false);
-        assert.equal(INSTANCE.nonBuiltInClass('java.math.BigDecimal'), false);
+        assert.equal(instance.nonBuiltInClass('BigDecimal'), false);
+        assert.equal(instance.nonBuiltInClass('java.math.BigDecimal'), false);
 
-        assert.equal(INSTANCE.nonBuiltInClass('String'), false);
-        assert.equal(INSTANCE.nonBuiltInClass('java.lang.String'), false);
+        assert.equal(instance.nonBuiltInClass('String'), false);
+        assert.equal(instance.nonBuiltInClass('java.lang.String'), false);
 
-        assert.equal(INSTANCE.nonBuiltInClass('Timestamp'), false);
-        assert.equal(INSTANCE.nonBuiltInClass('java.sql.Timestamp'), false);
+        assert.equal(instance.nonBuiltInClass('Timestamp'), false);
+        assert.equal(instance.nonBuiltInClass('java.sql.Timestamp'), false);
 
-        assert.equal(INSTANCE.nonBuiltInClass('Date'), false);
-        assert.equal(INSTANCE.nonBuiltInClass('java.sql.Date'), false);
+        assert.equal(instance.nonBuiltInClass('Date'), false);
+        assert.equal(instance.nonBuiltInClass('java.sql.Date'), false);
 
-        assert.equal(INSTANCE.nonBuiltInClass('Date'), false);
-        assert.equal(INSTANCE.nonBuiltInClass('java.util.Date'), false);
+        assert.equal(instance.nonBuiltInClass('Date'), false);
+        assert.equal(instance.nonBuiltInClass('java.util.Date'), false);
 
-        assert.equal(INSTANCE.nonBuiltInClass('CustomClass'), true);
-        assert.equal(INSTANCE.nonBuiltInClass('java.util.CustomClass'), true);
-        assert.equal(INSTANCE.nonBuiltInClass('my.package.CustomClass'), true);
+        assert.equal(instance.nonBuiltInClass('CustomClass'), true);
+        assert.equal(instance.nonBuiltInClass('java.util.CustomClass'), true);
+        assert.equal(instance.nonBuiltInClass('my.package.CustomClass'), true);
     });
 
     test('nonEnum', () => {
-        assert.equal(INSTANCE.nonEnum('org.apache.ignite.cache.CacheMode'), false);
-        assert.equal(INSTANCE.nonEnum('org.apache.ignite.transactions.TransactionConcurrency'), false);
-        assert.equal(INSTANCE.nonEnum('org.apache.ignite.cache.CacheWriteSynchronizationMode'), false);
-        assert.equal(INSTANCE.nonEnum('org.apache.ignite.igfs.IgfsIpcEndpointType'), false);
-        assert.equal(INSTANCE.nonEnum('java.io.Serializable'), true);
-        assert.equal(INSTANCE.nonEnum('BigDecimal'), true);
+        assert.equal(instance.nonEnum('org.apache.ignite.cache.CacheMode'), false);
+        assert.equal(instance.nonEnum('org.apache.ignite.transactions.TransactionConcurrency'), false);
+        assert.equal(instance.nonEnum('org.apache.ignite.cache.CacheWriteSynchronizationMode'), false);
+        assert.equal(instance.nonEnum('org.apache.ignite.igfs.IgfsIpcEndpointType'), false);
+        assert.equal(instance.nonEnum('java.io.Serializable'), true);
+        assert.equal(instance.nonEnum('BigDecimal'), true);
     });
 
     test('shortClassName', () => {
-        assert.equal(INSTANCE.shortClassName('java.math.BigDecimal'), 'BigDecimal');
-        assert.equal(INSTANCE.shortClassName('BigDecimal'), 'BigDecimal');
-        assert.equal(INSTANCE.shortClassName('int'), 'int');
-        assert.equal(INSTANCE.shortClassName('java.lang.Integer'), 'Integer');
-        assert.equal(INSTANCE.shortClassName('Integer'), 'Integer');
-        assert.equal(INSTANCE.shortClassName('java.util.UUID'), 'UUID');
-        assert.equal(INSTANCE.shortClassName('java.sql.Date'), 'Date');
-        assert.equal(INSTANCE.shortClassName('Date'), 'Date');
-        assert.equal(INSTANCE.shortClassName('com.my.Abstract'), 'Abstract');
-        assert.equal(INSTANCE.shortClassName('Abstract'), 'Abstract');
+        assert.equal(instance.shortClassName('java.math.BigDecimal'), 'BigDecimal');
+        assert.equal(instance.shortClassName('BigDecimal'), 'BigDecimal');
+        assert.equal(instance.shortClassName('int'), 'int');
+        assert.equal(instance.shortClassName('java.lang.Integer'), 'Integer');
+        assert.equal(instance.shortClassName('Integer'), 'Integer');
+        assert.equal(instance.shortClassName('java.util.UUID'), 'UUID');
+        assert.equal(instance.shortClassName('java.sql.Date'), 'Date');
+        assert.equal(instance.shortClassName('Date'), 'Date');
+        assert.equal(instance.shortClassName('com.my.Abstract'), 'Abstract');
+        assert.equal(instance.shortClassName('Abstract'), 'Abstract');
     });
 
     test('fullClassName', () => {
-        assert.equal(INSTANCE.fullClassName('BigDecimal'), 'java.math.BigDecimal');
+        assert.equal(instance.fullClassName('BigDecimal'), 'java.math.BigDecimal');
     });
 
     test('validIdentifier', () => {
-        assert.equal(INSTANCE.validIdentifier('myIdent'), true);
-        assert.equal(INSTANCE.validIdentifier('java.math.BigDecimal'), false);
-        assert.equal(INSTANCE.validIdentifier('2Demo'), false);
-        assert.equal(INSTANCE.validIdentifier('abra kadabra'), false);
-        assert.equal(INSTANCE.validIdentifier(undefined), false);
-        assert.equal(INSTANCE.validIdentifier(null), false);
-        assert.equal(INSTANCE.validIdentifier(''), false);
-        assert.equal(INSTANCE.validIdentifier(' '), false);
+        assert.equal(instance.validIdentifier('myIdent'), true);
+        assert.equal(instance.validIdentifier('java.math.BigDecimal'), false);
+        assert.equal(instance.validIdentifier('2Demo'), false);
+        assert.equal(instance.validIdentifier('abra kadabra'), false);
+        assert.equal(instance.validIdentifier(), false);
+        assert.equal(instance.validIdentifier(null), false);
+        assert.equal(instance.validIdentifier(''), false);
+        assert.equal(instance.validIdentifier(' '), false);
     });
 
     test('validClassName', () => {
-        assert.equal(INSTANCE.validClassName('java.math.BigDecimal'), true);
-        assert.equal(INSTANCE.validClassName('2Demo'), false);
-        assert.equal(INSTANCE.validClassName('abra kadabra'), false);
-        assert.equal(INSTANCE.validClassName(undefined), false);
-        assert.equal(INSTANCE.validClassName(null), false);
-        assert.equal(INSTANCE.validClassName(''), false);
-        assert.equal(INSTANCE.validClassName(' '), false);
+        assert.equal(instance.validClassName('java.math.BigDecimal'), true);
+        assert.equal(instance.validClassName('2Demo'), false);
+        assert.equal(instance.validClassName('abra kadabra'), false);
+        assert.equal(instance.validClassName(), false);
+        assert.equal(instance.validClassName(null), false);
+        assert.equal(instance.validClassName(''), false);
+        assert.equal(instance.validClassName(' '), false);
     });
 
     test('validPackage', () => {
-        assert.equal(INSTANCE.validPackage('java.math.BigDecimal'), true);
-        assert.equal(INSTANCE.validPackage('my.org.SomeClass'), true);
-        assert.equal(INSTANCE.validPackage('25'), false);
-        assert.equal(INSTANCE.validPackage('abra kadabra'), false);
-        assert.equal(INSTANCE.validPackage(''), false);
-        assert.equal(INSTANCE.validPackage(' '), false);
+        assert.equal(instance.validPackage('java.math.BigDecimal'), true);
+        assert.equal(instance.validPackage('my.org.SomeClass'), true);
+        assert.equal(instance.validPackage('25'), false);
+        assert.equal(instance.validPackage('abra kadabra'), false);
+        assert.equal(instance.validPackage(''), false);
+        assert.equal(instance.validPackage(' '), false);
     });
 
     test('packageSpecified', () => {
-        assert.equal(INSTANCE.packageSpecified('java.math.BigDecimal'), true);
-        assert.equal(INSTANCE.packageSpecified('BigDecimal'), false);
+        assert.equal(instance.packageSpecified('java.math.BigDecimal'), true);
+        assert.equal(instance.packageSpecified('BigDecimal'), false);
     });
 
     test('isKeyword', () => {
-        assert.equal(INSTANCE.isKeyword('abstract'), true);
-        assert.equal(INSTANCE.isKeyword('Abstract'), true);
-        assert.equal(INSTANCE.isKeyword('abra kadabra'), false);
-        assert.equal(INSTANCE.isKeyword(undefined), false);
-        assert.equal(INSTANCE.isKeyword(null), false);
-        assert.equal(INSTANCE.isKeyword(''), false);
-        assert.equal(INSTANCE.isKeyword(' '), false);
+        assert.equal(instance.isKeyword('abstract'), true);
+        assert.equal(instance.isKeyword('Abstract'), true);
+        assert.equal(instance.isKeyword('abra kadabra'), false);
+        assert.equal(instance.isKeyword(), false);
+        assert.equal(instance.isKeyword(null), false);
+        assert.equal(instance.isKeyword(''), false);
+        assert.equal(instance.isKeyword(' '), false);
     });
 
     test('isPrimitive', () => {
-        assert.equal(INSTANCE.isPrimitive('boolean'), true);
+        assert.equal(instance.isPrimitive('boolean'), true);
     });
 
     test('validUUID', () => {
-        assert.equal(INSTANCE.validUUID('123e4567-e89b-12d3-a456-426655440000'), true);
-        assert.equal(INSTANCE.validUUID('12345'), false);
-        assert.equal(INSTANCE.validUUID(undefined), false);
-        assert.equal(INSTANCE.validUUID(null), false);
-        assert.equal(INSTANCE.validUUID(''), false);
-        assert.equal(INSTANCE.validUUID(' '), false);
+        assert.equal(instance.validUUID('123e4567-e89b-12d3-a456-426655440000'), true);
+        assert.equal(instance.validUUID('12345'), false);
+        assert.equal(instance.validUUID(), false);
+        assert.equal(instance.validUUID(null), false);
+        assert.equal(instance.validUUID(''), false);
+        assert.equal(instance.validUUID(' '), false);
     });
 });
diff --git a/modules/web-console/frontend/test/unit/SharpTransformer.test.js b/modules/web-console/frontend/test/unit/SharpTransformer.test.js
index 20de266af6bb1..3fb9265b73c55 100644
--- a/modules/web-console/frontend/test/unit/SharpTransformer.test.js
+++ b/modules/web-console/frontend/test/unit/SharpTransformer.test.js
@@ -18,7 +18,7 @@
 import generator from '../../app/modules/configuration/generator/PlatformGenerator';
 import transformer from '../../app/modules/configuration/generator/SharpTransformer.service';
 
-import { assert } from 'chai';
+import { suite, test } from 'mocha';
 
 suite.skip('SharpTransformerTestsSuite', () => {
     test('AtomicConfiguration', () => {
diff --git a/modules/web-console/frontend/test/unit/SpringTransformer.test.js b/modules/web-console/frontend/test/unit/SpringTransformer.test.js
index 7998f666befe4..e96cab3c2d2f4 100644
--- a/modules/web-console/frontend/test/unit/SpringTransformer.test.js
+++ b/modules/web-console/frontend/test/unit/SpringTransformer.test.js
@@ -20,7 +20,7 @@ import JavaTypes from '../../app/services/JavaTypes.service.js';
 import generator from '../../app/modules/configuration/generator/ConfigurationGenerator';
 import transformer from '../../app/modules/configuration/generator/SpringTransformer.service';
 
-import { assert } from 'chai';
+import { suite, test } from 'mocha';
 
 suite.skip('SpringTransformerTestsSuite', () => {
     test('AtomicConfiguration', () => {
diff --git a/modules/web-console/frontend/test/unit/SqlTypes.test.js b/modules/web-console/frontend/test/unit/SqlTypes.test.js
index 2d54bdfdb1f25..17f899cf90900 100644
--- a/modules/web-console/frontend/test/unit/SqlTypes.test.js
+++ b/modules/web-console/frontend/test/unit/SqlTypes.test.js
@@ -19,6 +19,7 @@ import SqlTypes from '../../app/services/SqlTypes.service.js';
 
 const INSTANCE = new SqlTypes();
 
+import { suite, test } from 'mocha';
 import { assert } from 'chai';
 
 suite('SqlTypesTestsSuite', () => {
@@ -27,7 +28,7 @@ suite('SqlTypesTestsSuite', () => {
         assert.equal(INSTANCE.validIdentifier('java.math.BigDecimal'), false);
         assert.equal(INSTANCE.validIdentifier('2Demo'), false);
         assert.equal(INSTANCE.validIdentifier('abra kadabra'), false);
-        assert.equal(INSTANCE.validIdentifier(undefined), false);
+        assert.equal(INSTANCE.validIdentifier(), false);
         assert.equal(INSTANCE.validIdentifier(null), false);
         assert.equal(INSTANCE.validIdentifier(''), false);
         assert.equal(INSTANCE.validIdentifier(' '), false);
@@ -38,7 +39,7 @@ suite('SqlTypesTestsSuite', () => {
         assert.equal(INSTANCE.isKeyword('Group'), true);
         assert.equal(INSTANCE.isKeyword('select'), true);
         assert.equal(INSTANCE.isKeyword('abra kadabra'), false);
-        assert.equal(INSTANCE.isKeyword(undefined), false);
+        assert.equal(INSTANCE.isKeyword(), false);
         assert.equal(INSTANCE.isKeyword(null), false);
         assert.equal(INSTANCE.isKeyword(''), false);
         assert.equal(INSTANCE.isKeyword(' '), false);
@@ -47,5 +48,5 @@ suite('SqlTypesTestsSuite', () => {
     test('findJdbcType', () => {
         assert.equal(INSTANCE.findJdbcType(0).dbName, 'NULL');
         assert.equal(INSTANCE.findJdbcType(5555).dbName, 'Unknown');
-    })
+    });
 });
diff --git a/modules/web-console/frontend/test/unit/UserAuth.test.js b/modules/web-console/frontend/test/unit/UserAuth.test.js
index dbba1f63fa747..7b6b24c2944f7 100644
--- a/modules/web-console/frontend/test/unit/UserAuth.test.js
+++ b/modules/web-console/frontend/test/unit/UserAuth.test.js
@@ -15,7 +15,9 @@
  * limitations under the License.
  */
 
-import AuthService from '../../app/modules/user/Auth.service';
+// import AuthService from '../../app/modules/user/Auth.service';
+
+import { suite, test } from 'mocha';
 
 suite('AuthServiceTestsSuite', () => {
     test('SignIn', (done) => {
diff --git a/modules/web-console/frontend/test/unit/Version.test.js b/modules/web-console/frontend/test/unit/Version.test.js
index 72685eae03239..d8c8f656367ff 100644
--- a/modules/web-console/frontend/test/unit/Version.test.js
+++ b/modules/web-console/frontend/test/unit/Version.test.js
@@ -19,6 +19,7 @@ import VersionService from '../../app/modules/configuration/Version.service';
 
 const INSTANCE = new VersionService();
 
+import { suite, test } from 'mocha';
 import { assert } from 'chai';
 
 suite('VersionServiceTestsSuite', () => {
diff --git a/modules/web-console/frontend/test/unit/defaultName.filter.test.js b/modules/web-console/frontend/test/unit/defaultName.filter.test.js
index 5f282900b4f97..2eaf2238e17ee 100644
--- a/modules/web-console/frontend/test/unit/defaultName.filter.test.js
+++ b/modules/web-console/frontend/test/unit/defaultName.filter.test.js
@@ -17,22 +17,25 @@
 
 import defaultName from '../../app/filters/default-name.filter';
 
+import { suite, test } from 'mocha';
 import { assert } from 'chai';
 
-const INSTANCE = defaultName[0]();
+const instance = defaultName[0]();
 
 suite('defaultName', () => {
     test('defaultName filter', () => {
-        assert.equal(INSTANCE(''), '<default>');
-        assert.equal(INSTANCE(null), '<default>');
-        assert.equal(INSTANCE(undefined), '<default>');
-        assert.equal(INSTANCE('', false), '<default>');
-        assert.equal(INSTANCE(null, false), '<default>');
-        assert.equal(INSTANCE(undefined, false), '<default>');
-        assert.equal(INSTANCE('', true), '&lt;default&gt;');
-        assert.equal(INSTANCE(null, true), '&lt;default&gt;');
-        assert.equal(INSTANCE(undefined, true), '&lt;default&gt;');
-        assert.equal(INSTANCE("name", false), 'name');
-        assert.equal(INSTANCE("name", true), 'name');
+        let undef;
+
+        assert.equal(instance(''), '<default>');
+        assert.equal(instance(null), '<default>');
+        assert.equal(instance(), '<default>');
+        assert.equal(instance('', false), '<default>');
+        assert.equal(instance(null, false), '<default>');
+        assert.equal(instance(undef, false), '<default>');
+        assert.equal(instance('', true), '&lt;default&gt;');
+        assert.equal(instance(null, true), '&lt;default&gt;');
+        assert.equal(instance(undef, true), '&lt;default&gt;');
+        assert.equal(instance('name', false), 'name');
+        assert.equal(instance('name', true), 'name');
     });
 });
diff --git a/modules/web-console/frontend/views/403.jade b/modules/web-console/frontend/views/403.pug
similarity index 100%
rename from modules/web-console/frontend/views/403.jade
rename to modules/web-console/frontend/views/403.pug
diff --git a/modules/web-console/frontend/views/404.jade b/modules/web-console/frontend/views/404.pug
similarity index 100%
rename from modules/web-console/frontend/views/404.jade
rename to modules/web-console/frontend/views/404.pug
diff --git a/modules/web-console/frontend/views/base.jade b/modules/web-console/frontend/views/base.pug
similarity index 94%
rename from modules/web-console/frontend/views/base.jade
rename to modules/web-console/frontend/views/base.pug
index a910d1b2d84b5..b05cdbabeaef1 100644
--- a/modules/web-console/frontend/views/base.jade
+++ b/modules/web-console/frontend/views/base.pug
@@ -14,9 +14,9 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include includes/header
+include ./includes/header
 
 .container.body-container
     .main-content(ui-view='')
 
-include includes/footer
+include ./includes/footer
diff --git a/modules/web-console/frontend/views/configuration/caches.jade b/modules/web-console/frontend/views/configuration/caches.tpl.pug
similarity index 89%
rename from modules/web-console/frontend/views/configuration/caches.jade
rename to modules/web-console/frontend/views/configuration/caches.tpl.pug
index 73a6309d223c6..fca41a895a36f 100644
--- a/modules/web-console/frontend/views/configuration/caches.jade
+++ b/modules/web-console/frontend/views/configuration/caches.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 .docs-header
     h1 Configure Ignite Caches
@@ -36,20 +36,20 @@ include /app/helpers/jade/mixins.jade
             div(bs-collapse='' data-allow-multiple='true' ng-model='ui.activePanels')
                 form.form-horizontal(name='ui.inputForm' novalidate ng-if='contentVisible()')
                     .panel-group
-                        include /app/modules/states/configuration/caches/general.jade
-                        include /app/modules/states/configuration/caches/memory.jade
-                        include /app/modules/states/configuration/caches/query.jade
-                        include /app/modules/states/configuration/caches/store.jade
+                        include /app/modules/states/configuration/caches/general
+                        include /app/modules/states/configuration/caches/memory
+                        include /app/modules/states/configuration/caches/query
+                        include /app/modules/states/configuration/caches/store
 
                         +advanced-options-toggle-default
 
                         div(ng-show='ui.expanded')
-                            include /app/modules/states/configuration/caches/affinity.jade
-                            include /app/modules/states/configuration/caches/concurrency.jade
-                            include /app/modules/states/configuration/caches/near-cache-client.jade
-                            include /app/modules/states/configuration/caches/near-cache-server.jade
-                            include /app/modules/states/configuration/caches/node-filter.jade
-                            include /app/modules/states/configuration/caches/rebalance.jade
-                            include /app/modules/states/configuration/caches/statistics.jade
+                            include /app/modules/states/configuration/caches/affinity
+                            include /app/modules/states/configuration/caches/concurrency
+                            include /app/modules/states/configuration/caches/near-cache-client
+                            include /app/modules/states/configuration/caches/near-cache-server
+                            include /app/modules/states/configuration/caches/node-filter
+                            include /app/modules/states/configuration/caches/rebalance
+                            include /app/modules/states/configuration/caches/statistics
 
                             +advanced-options-toggle-default
diff --git a/modules/web-console/frontend/views/configuration/clusters.jade b/modules/web-console/frontend/views/configuration/clusters.tpl.pug
similarity index 83%
rename from modules/web-console/frontend/views/configuration/clusters.jade
rename to modules/web-console/frontend/views/configuration/clusters.tpl.pug
index 78264647b800f..c97901260f243 100644
--- a/modules/web-console/frontend/views/configuration/clusters.jade
+++ b/modules/web-console/frontend/views/configuration/clusters.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 .docs-header
     h1 Configure Ignite Clusters
@@ -36,33 +36,33 @@ include /app/helpers/jade/mixins.jade
             div(bs-collapse='' data-allow-multiple='true' ng-model='ui.activePanels')
                 form.form-horizontal(name='ui.inputForm' novalidate ng-if='contentVisible()')
                     .panel-group
-                        include /app/modules/states/configuration/clusters/general.jade
+                        include /app/modules/states/configuration/clusters/general
 
                         +advanced-options-toggle-default
 
                         div(ng-show='ui.expanded')
-                            include /app/modules/states/configuration/clusters/atomic.jade
-                            include /app/modules/states/configuration/clusters/binary.jade
-                            include /app/modules/states/configuration/clusters/cache-key-cfg.jade
-                            include /app/modules/states/configuration/clusters/checkpoint.jade
-                            include /app/modules/states/configuration/clusters/collision.jade
-                            include /app/modules/states/configuration/clusters/communication.jade
-                            include /app/modules/states/configuration/clusters/connector.jade
-                            include /app/modules/states/configuration/clusters/deployment.jade
-                            include /app/modules/states/configuration/clusters/discovery.jade
-                            include /app/modules/states/configuration/clusters/events.jade
-                            include /app/modules/states/configuration/clusters/failover.jade
-                            include /app/modules/states/configuration/clusters/igfs.jade
-                            include /app/modules/states/configuration/clusters/load-balancing.jade
-                            include /app/modules/states/configuration/clusters/logger.jade
-                            include /app/modules/states/configuration/clusters/marshaller.jade
-                            include /app/modules/states/configuration/clusters/metrics.jade
-                            include /app/modules/states/configuration/clusters/odbc.jade
-                            include /app/modules/states/configuration/clusters/ssl.jade
-                            include /app/modules/states/configuration/clusters/swap.jade
-                            include /app/modules/states/configuration/clusters/thread.jade
-                            include /app/modules/states/configuration/clusters/time.jade
-                            include /app/modules/states/configuration/clusters/transactions.jade
-                            include /app/modules/states/configuration/clusters/attributes.jade
+                            include /app/modules/states/configuration/clusters/atomic
+                            include /app/modules/states/configuration/clusters/binary
+                            include /app/modules/states/configuration/clusters/cache-key-cfg
+                            include /app/modules/states/configuration/clusters/checkpoint
+                            include /app/modules/states/configuration/clusters/collision
+                            include /app/modules/states/configuration/clusters/communication
+                            include /app/modules/states/configuration/clusters/connector
+                            include /app/modules/states/configuration/clusters/deployment
+                            include /app/modules/states/configuration/clusters/discovery
+                            include /app/modules/states/configuration/clusters/events
+                            include /app/modules/states/configuration/clusters/failover
+                            include /app/modules/states/configuration/clusters/igfs
+                            include /app/modules/states/configuration/clusters/load-balancing
+                            include /app/modules/states/configuration/clusters/logger
+                            include /app/modules/states/configuration/clusters/marshaller
+                            include /app/modules/states/configuration/clusters/metrics
+                            include /app/modules/states/configuration/clusters/odbc
+                            include /app/modules/states/configuration/clusters/ssl
+                            include /app/modules/states/configuration/clusters/swap
+                            include /app/modules/states/configuration/clusters/thread
+                            include /app/modules/states/configuration/clusters/time
+                            include /app/modules/states/configuration/clusters/transactions
+                            include /app/modules/states/configuration/clusters/attributes
 
                             +advanced-options-toggle-default
diff --git a/modules/web-console/frontend/views/configuration/domains-import.jade b/modules/web-console/frontend/views/configuration/domains-import.tpl.pug
similarity index 99%
rename from modules/web-console/frontend/views/configuration/domains-import.jade
rename to modules/web-console/frontend/views/configuration/domains-import.tpl.pug
index bbcb391cbedef..baaa2d34cf1ad 100644
--- a/modules/web-console/frontend/views/configuration/domains-import.jade
+++ b/modules/web-console/frontend/views/configuration/domains-import.tpl.pug
@@ -14,13 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 mixin chk(mdl, change, tip)
     input(type='checkbox' ng-model=mdl ng-change=change bs-tooltip='' data-title=tip data-trigger='hover' data-placement='top')
 
 mixin td-ellipses-lbl(w, lbl)
-    td.td-ellipsis(width='#{w}' style='min-width: #{w}; max-width: #{w}')
+    td.td-ellipsis(width=`${w}` style=`min-width: ${w}; max-width: ${w}`)
         label #{lbl}
 
 .modal.modal-domain-import.center(role='dialog')
diff --git a/modules/web-console/frontend/views/configuration/domains.jade b/modules/web-console/frontend/views/configuration/domains.tpl.pug
similarity index 97%
rename from modules/web-console/frontend/views/configuration/domains.jade
rename to modules/web-console/frontend/views/configuration/domains.tpl.pug
index 14e9ca5240ea0..33528c7c8dd68 100644
--- a/modules/web-console/frontend/views/configuration/domains.jade
+++ b/modules/web-console/frontend/views/configuration/domains.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 .docs-header
     h1 Configure Domain Model And SQL Queries
@@ -61,6 +61,6 @@ include /app/helpers/jade/mixins.jade
             div(bs-collapse='' data-allow-multiple='true' ng-model='ui.activePanels')
                 form.form-horizontal(name='ui.inputForm' novalidate ng-if='contentVisible()')
                     .panel-group
-                        include /app/modules/states/configuration/domains/general.jade
-                        include /app/modules/states/configuration/domains/query.jade
-                        include /app/modules/states/configuration/domains/store.jade
+                        include /app/modules/states/configuration/domains/general
+                        include /app/modules/states/configuration/domains/query
+                        include /app/modules/states/configuration/domains/store
diff --git a/modules/web-console/frontend/views/configuration/igfs.jade b/modules/web-console/frontend/views/configuration/igfs.tpl.pug
similarity index 94%
rename from modules/web-console/frontend/views/configuration/igfs.jade
rename to modules/web-console/frontend/views/configuration/igfs.tpl.pug
index 10c902be3b3be..89e09209d0cc3 100644
--- a/modules/web-console/frontend/views/configuration/igfs.jade
+++ b/modules/web-console/frontend/views/configuration/igfs.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 .docs-header
     h1 Configure Ignite In-memory File Systems
@@ -37,15 +37,15 @@ include /app/helpers/jade/mixins.jade
             div(bs-collapse='' data-allow-multiple='true' ng-model='ui.activePanels')
                 form.form-horizontal(name='ui.inputForm' novalidate ng-if='contentVisible()')
                     .panel-group
-                        include /app/modules/states/configuration/igfs/general.jade
+                        include /app/modules/states/configuration/igfs/general
 
                         +advanced-options-toggle-default
 
                         div(ng-show='ui.expanded')
-                            include /app/modules/states/configuration/igfs/secondary.jade
-                            include /app/modules/states/configuration/igfs/ipc.jade
-                            include /app/modules/states/configuration/igfs/fragmentizer.jade
-                            include /app/modules/states/configuration/igfs/dual.jade
-                            include /app/modules/states/configuration/igfs/misc.jade
+                            include /app/modules/states/configuration/igfs/secondary
+                            include /app/modules/states/configuration/igfs/ipc
+                            include /app/modules/states/configuration/igfs/fragmentizer
+                            include /app/modules/states/configuration/igfs/dual
+                            include /app/modules/states/configuration/igfs/misc
 
                             +advanced-options-toggle-default
diff --git a/modules/web-console/frontend/views/configuration/sidebar.jade b/modules/web-console/frontend/views/configuration/sidebar.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/configuration/sidebar.jade
rename to modules/web-console/frontend/views/configuration/sidebar.tpl.pug
diff --git a/modules/web-console/frontend/views/configuration/summary-project-structure.jade b/modules/web-console/frontend/views/configuration/summary-project-structure.tpl.pug
similarity index 99%
rename from modules/web-console/frontend/views/configuration/summary-project-structure.jade
rename to modules/web-console/frontend/views/configuration/summary-project-structure.tpl.pug
index 29d4538c7cb90..31a557fca15da 100644
--- a/modules/web-console/frontend/views/configuration/summary-project-structure.jade
+++ b/modules/web-console/frontend/views/configuration/summary-project-structure.tpl.pug
@@ -13,6 +13,7 @@
     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     See the License for the specific language governing permissions and
     limitations under the License.
+
 .popover.summary-project-structure
     h3.popover-title
         label.labelField Project structure
diff --git a/modules/web-console/frontend/views/configuration/summary-tabs.jade b/modules/web-console/frontend/views/configuration/summary-tabs.pug
similarity index 97%
rename from modules/web-console/frontend/views/configuration/summary-tabs.jade
rename to modules/web-console/frontend/views/configuration/summary-tabs.pug
index 847b42f70b2f5..d05e3a9eae43b 100644
--- a/modules/web-console/frontend/views/configuration/summary-tabs.jade
+++ b/modules/web-console/frontend/views/configuration/summary-tabs.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-ul.nav(ng-class='$navClass', role='tablist')
+ul.nav(ng-class='$navClass' role='tablist')
     li(role='presentation' ng-repeat='$pane in $panes track by $index' ng-class='[ $isActive($pane, $index) ? $activeClass : "", $pane.disabled ? "disabled" : "" ]')
         a.summary-tab(ng-show='$pane.title != "POJO" || (cluster | hasPojo)' ng-switch='$pane.title' role='tab' data-toggle='tab' ng-click='!$pane.disabled && $setActive($pane.name || $index)' data-index='{{ $index }}' aria-controls='$pane.title') {{$pane.title}}
             img(ng-switch-when='XML' src='/images/xml.png')
diff --git a/modules/web-console/frontend/views/configuration/summary.jade b/modules/web-console/frontend/views/configuration/summary.tpl.pug
similarity index 93%
rename from modules/web-console/frontend/views/configuration/summary.jade
rename to modules/web-console/frontend/views/configuration/summary.tpl.pug
index a04f0dbbf074b..9fdd0de3f3097 100644
--- a/modules/web-console/frontend/views/configuration/summary.jade
+++ b/modules/web-console/frontend/views/configuration/summary.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 mixin hard-link(ref, txt)
     a(style='color:#ec1c24' href=ref target='_blank') #{txt}
@@ -44,7 +44,7 @@ mixin hard-link(ref, txt)
                             i.fa.fa-fw.fa-refresh.fa-spin(ng-show='isPrepareDownloading')
                             span.tipLabel Download project
                     button.btn.btn-primary(bs-tooltip='' data-title='Preview generated project structure' data-placement='bottom')
-                        div(bs-popover data-template-url='/configuration/summary-project-structure.html', data-placement='bottom', data-trigger='click' data-auto-close='true')
+                        div(bs-popover data-template-url='{{ ctrl.summaryProjectStructureTemplateUrl }}', data-placement='bottom', data-trigger='click' data-auto-close='true')
                             i.fa.fa-sitemap
                             label.tipLabel Project structure
                     button.btn.btn-primary(id='proprietary-jdbc-drivers' ng-if='downloadJdbcDriversVisible()' ng-click='downloadJdbcDrivers()' bs-tooltip='' data-title='Open proprietary JDBC drivers download pages' data-placement='bottom') Download JDBC drivers
@@ -61,7 +61,7 @@ mixin hard-link(ref, txt)
 
                     .panel-collapse(id='server' role='tabpanel' bs-collapse-target)
                         .summary-tabs(ignite-ui-ace-tabs)
-                            div(bs-tabs data-bs-active-pane="tabsServer.activeTab" template='configuration/summary-tabs.html')
+                            div(bs-tabs data-bs-active-pane='tabsServer.activeTab' data-template='summary-tabs.html')
                                 div(bs-pane title='XML')
                                     ignite-ui-ace-spring(ng-if='tabsServer.activeTab == 0 || tabsServer.init[0]' ng-init='tabsServer.init[0] = true' data-master='cluster' data-generator='igniteConfiguration' data-no-deep-watch)
                                 div(bs-pane title='Java')
@@ -78,7 +78,7 @@ mixin hard-link(ref, txt)
 
                     .panel-collapse(id='client' role='tabpanel' bs-collapse-target)
                         .summary-tabs(ignite-ui-ace-tabs)
-                            div(bs-tabs data-bs-active-pane="tabsClient.activeTab" template='configuration/summary-tabs.html')
+                            div(bs-tabs data-bs-active-pane='tabsClient.activeTab' data-template='summary-tabs.html')
                                 div(bs-pane title='XML')
                                     ignite-ui-ace-spring(ng-if='tabsClient.activeTab == 0 || tabsClient.init[0]' ng-init='tabsClient.init[0] = true' data-master='cluster' data-generator='igniteConfiguration' data-client='true' data-no-deep-watch)
                                 div(bs-pane title='Java')
diff --git a/modules/web-console/frontend/views/includes/footer.jade b/modules/web-console/frontend/views/includes/footer.pug
similarity index 100%
rename from modules/web-console/frontend/views/includes/footer.jade
rename to modules/web-console/frontend/views/includes/footer.pug
diff --git a/modules/web-console/frontend/views/includes/header.jade b/modules/web-console/frontend/views/includes/header.pug
similarity index 100%
rename from modules/web-console/frontend/views/includes/header.jade
rename to modules/web-console/frontend/views/includes/header.pug
diff --git a/modules/web-console/frontend/views/index.jade b/modules/web-console/frontend/views/index.pug
similarity index 100%
rename from modules/web-console/frontend/views/index.jade
rename to modules/web-console/frontend/views/index.pug
diff --git a/modules/web-console/frontend/views/reset.jade b/modules/web-console/frontend/views/reset.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/reset.jade
rename to modules/web-console/frontend/views/reset.tpl.pug
diff --git a/modules/web-console/frontend/views/settings/admin.jade b/modules/web-console/frontend/views/settings/admin.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/settings/admin.jade
rename to modules/web-console/frontend/views/settings/admin.tpl.pug
diff --git a/modules/web-console/frontend/views/settings/profile.jade b/modules/web-console/frontend/views/settings/profile.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/settings/profile.jade
rename to modules/web-console/frontend/views/settings/profile.tpl.pug
diff --git a/modules/web-console/frontend/views/signin.jade b/modules/web-console/frontend/views/signin.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/signin.jade
rename to modules/web-console/frontend/views/signin.tpl.pug
diff --git a/modules/web-console/frontend/views/sql/cache-metadata.jade b/modules/web-console/frontend/views/sql/cache-metadata.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/sql/cache-metadata.jade
rename to modules/web-console/frontend/views/sql/cache-metadata.tpl.pug
diff --git a/modules/web-console/frontend/views/sql/chart-settings.jade b/modules/web-console/frontend/views/sql/chart-settings.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/sql/chart-settings.jade
rename to modules/web-console/frontend/views/sql/chart-settings.tpl.pug
diff --git a/modules/web-console/frontend/views/sql/notebook-new.jade b/modules/web-console/frontend/views/sql/notebook-new.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/sql/notebook-new.jade
rename to modules/web-console/frontend/views/sql/notebook-new.tpl.pug
diff --git a/modules/web-console/frontend/views/sql/paragraph-rate.jade b/modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/sql/paragraph-rate.jade
rename to modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug
diff --git a/modules/web-console/frontend/views/sql/sql.jade b/modules/web-console/frontend/views/sql/sql.tpl.pug
similarity index 95%
rename from modules/web-console/frontend/views/sql/sql.jade
rename to modules/web-console/frontend/views/sql/sql.tpl.pug
index 61d5b3037825a..931d6327888d3 100644
--- a/modules/web-console/frontend/views/sql/sql.jade
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
-include /app/components/ui-grid-settings/ui-grid-settings.jade
+include /app/helpers/jade/mixins
+include /app/components/ui-grid-settings/ui-grid-settings
 
 mixin btn-toolbar(btn, click, tip, focusId)
     i.btn.btn-default.fa(class=btn ng-click=click bs-tooltip='' data-title=tip ignite-on-click-focus=focusId data-trigger='hover' data-placement='bottom')
 
 mixin btn-toolbar-data(btn, kind, tip)
-    i.btn.btn-default.fa(class=btn ng-click='setResult(paragraph, "#{kind}")' ng-class='{active: resultEq(paragraph, "#{kind}")}' bs-tooltip='' data-title=tip data-trigger='hover' data-placement='bottom')
+    i.btn.btn-default.fa(class=btn ng-click=`setResult(paragraph, '${kind}')` ng-class=`{active: resultEq(paragraph, '${kind}')}` bs-tooltip='' data-title=tip data-trigger='hover' data-placement='bottom')
 
 mixin result-toolbar
     .btn-group(ng-model='paragraph.result' ng-click='$event.stopPropagation()' style='left: 50%; margin: 0 0 0 -70px;display: block;')
@@ -35,7 +35,7 @@ mixin chart-settings
     .total.row
         .col-xs-4
             .chart-settings-link(ng-show='paragraph.chart && paragraph.chartColumns.length > 0')
-                a(title='Click to show chart settings dialog' ng-click='$event.stopPropagation()' bs-popover data-template-url='/sql/chart-settings.html' data-placement='bottom' data-auto-close='1' data-trigger='click')
+                a(title='Click to show chart settings dialog' ng-click='$event.stopPropagation()' bs-popover data-template-url='{{ $ctrl.chartSettingsTemplateUrl }}' data-placement='bottom' data-auto-close='1' data-trigger='click')
                     i.fa.fa-bars
                     | Chart settings
                 div(ng-show='paragraphTimeSpanVisible(paragraph)')
@@ -92,7 +92,7 @@ mixin paragraph-rename
 
 mixin query-settings
     label.tipLabel(bs-tooltip data-placement='bottom' data-title='Configure periodical execution of last successfully executed query') Refresh rate:
-        button.btn.btn-default.fa.fa-clock-o.tipLabel(ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='/sql/paragraph-rate.html' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
+        button.btn.btn-default.fa.fa-clock-o.tipLabel(ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='{{ $ctrl.paragraphRateTemplateUrl }}' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
 
     label.tipLabel(bs-tooltip data-placement='bottom' data-title='Max number of rows to show in query result as one page') Page size:
         button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-select bs-options='item for item in pageSizes')
@@ -201,7 +201,7 @@ mixin paragraph-scan
 
                 -var nextVisibleCondition = 'paragraph.resultType() != "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
 
-                .pull-right(ng-show='#{nextVisibleCondition}' ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
+                .pull-right(ng-show=`${nextVisibleCondition}` ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
                     i.fa.fa-chevron-circle-right
                     a Next
 
@@ -217,7 +217,7 @@ mixin paragraph-query
             .col-xs-4.col-sm-3
                 div(ng-show='caches.length > 0' style='padding: 5px 10px' st-table='displayedCaches' st-safe-src='caches')
                     lable.labelField.labelFormField Caches:
-                    i.fa.fa-database.tipField(title='Click to show cache types metadata dialog' bs-popover data-template-url='/sql/cache-metadata.html' data-placement='bottom' data-trigger='click' data-container='#{{ paragraph.id }}')
+                    i.fa.fa-database.tipField(title='Click to show cache types metadata dialog' bs-popover data-template-url='{{ $ctrl.cacheMetadataTemplateUrl }}' data-placement='bottom' data-trigger='click' data-container='#{{ paragraph.id }}')
                     .input-tip
                         input.form-control(type='text' st-search='label' placeholder='Filter caches...')
                     table.links
@@ -248,11 +248,11 @@ mixin paragraph-query
 
                 -var nextVisibleCondition = 'paragraph.resultType() != "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
 
-                .pull-right(ng-show='#{nextVisibleCondition}' ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
+                .pull-right(ng-show=`${nextVisibleCondition}` ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
                     i.fa.fa-chevron-circle-right
                     a Next
 
-.row(ng-controller='sqlController')
+.row
     .docs-content
         .row(ng-if='notebook' bs-affix style='margin-bottom: 20px;')
             +notebook-rename
diff --git a/modules/web-console/frontend/views/templates/agent-download.jade b/modules/web-console/frontend/views/templates/agent-download.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/agent-download.jade
rename to modules/web-console/frontend/views/templates/agent-download.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/alert.jade b/modules/web-console/frontend/views/templates/alert.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/alert.jade
rename to modules/web-console/frontend/views/templates/alert.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/batch-confirm.jade b/modules/web-console/frontend/views/templates/batch-confirm.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/batch-confirm.jade
rename to modules/web-console/frontend/views/templates/batch-confirm.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/clone.jade b/modules/web-console/frontend/views/templates/clone.tpl.pug
similarity index 98%
rename from modules/web-console/frontend/views/templates/clone.jade
rename to modules/web-console/frontend/views/templates/clone.tpl.pug
index 99ec58be4697d..e62cb18a8fb60 100644
--- a/modules/web-console/frontend/views/templates/clone.jade
+++ b/modules/web-console/frontend/views/templates/clone.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include /app/helpers/jade/mixins.jade
+include /app/helpers/jade/mixins
 
 .modal(tabindex='-1' role='dialog')
     .modal-dialog
diff --git a/modules/web-console/frontend/views/templates/confirm.jade b/modules/web-console/frontend/views/templates/confirm.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/confirm.jade
rename to modules/web-console/frontend/views/templates/confirm.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/demo-info.jade b/modules/web-console/frontend/views/templates/demo-info.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/demo-info.jade
rename to modules/web-console/frontend/views/templates/demo-info.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/dropdown.jade b/modules/web-console/frontend/views/templates/dropdown.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/dropdown.jade
rename to modules/web-console/frontend/views/templates/dropdown.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/getting-started.jade b/modules/web-console/frontend/views/templates/getting-started.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/getting-started.jade
rename to modules/web-console/frontend/views/templates/getting-started.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/message.jade b/modules/web-console/frontend/views/templates/message.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/message.jade
rename to modules/web-console/frontend/views/templates/message.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/pagination.jade b/modules/web-console/frontend/views/templates/pagination.jade
deleted file mode 100644
index 08ced60f3500a..0000000000000
--- a/modules/web-console/frontend/views/templates/pagination.jade
+++ /dev/null
@@ -1,32 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-nav(ng-if='numPages && pages.length >= 2')
-    ul.pagination
-        li(ng-if='currentPage > 1')
-            a(href='javascript:void(0);' ng-click='selectPage(1)' bs-tooltip='' data-title='First page' data-placement='bottom')
-                i.fa.fa-angle-double-left
-        li(ng-if='currentPage > 1')
-            a(href='javascript:void(0);' ng-click='selectPage(currentPage-1)' bs-tooltip='' data-title='Previous page' data-placement='bottom')
-                i.fa.fa-angle-left
-        li(ng-repeat='page in pages' ng-class='{active: page==currentPage}')
-            a(href='javascript:void(0);' ng-click='selectPage(page)') {{page}}
-        li(ng-if='currentPage < numPages')
-            a(href='javascript:void(0);' ng-click='selectPage(currentPage+1)' bs-tooltip='' data-title='Next page' data-placement='bottom')
-                i.fa.fa-angle-right
-        li(ng-if='currentPage < numPages')
-            a(href='javascript:void(0);' ng-click='selectPage(numPages)' bs-tooltip='' data-title='Last page' data-placement='bottom')
-                i.fa.fa-angle-double-right
\ No newline at end of file
diff --git a/modules/web-console/frontend/views/templates/select.jade b/modules/web-console/frontend/views/templates/select.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/select.jade
rename to modules/web-console/frontend/views/templates/select.tpl.pug
diff --git a/modules/web-console/frontend/views/templates/validation-error.jade b/modules/web-console/frontend/views/templates/validation-error.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/templates/validation-error.jade
rename to modules/web-console/frontend/views/templates/validation-error.tpl.pug

From 1c05eb30797ab9862c87e9d11237e7bcfb9782a0 Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgain.com>
Date: Thu, 9 Mar 2017 10:00:34 +0700
Subject: [PATCH 054/516] IGNITE_4659 Minor templates fixes. (cherry picked
 from commit 2d385c5)

---
 .../app/modules/states/configuration/clusters/binary.pug        | 2 +-
 .../app/modules/states/configuration/clusters/communication.pug | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
index ee9f37bd1645b..2e14e7fc29803 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
@@ -72,6 +72,6 @@ include /app/helpers/jade/mixins
                                 +checkbox('Enum', 'model.enum', 'enum', 'Flag indicating that this type is the enum')
 
                 .settings-row
-                    +checkbox('Compact footer', model + '.compactFooter', '"compactFooter"', 'When enabled, Ignite will not write fields metadata when serializing objects(this will increase serialization performance), because internally #[b BinaryMarshaller] already distribute metadata inside cluster')
+                    +checkbox('Compact footer', model + '.compactFooter', '"compactFooter"', 'When enabled, Ignite will not write fields metadata when serializing objects (this will increase serialization performance), because internally <b>BinaryMarshaller</b> already distribute metadata inside cluster')
             .col-sm-6
                 +preview-xml-java(model, 'clusterBinary')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
index cd93565720e2b..93bb5cea34d7c 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
@@ -53,7 +53,7 @@ include /app/helpers/jade/mixins
                 .settings-row
                     +number-min-max('Shared memory port:', `${communication}.sharedMemoryPort`, '"sharedMemoryPort"', 'true', '48100', '-1', '65535',
                         'Local port to accept shared memory connections<br/>\
-                        If set to #[b -1] shared memory communication will be disabled')
+                        If set to <b>-1</b> shared memory communication will be disabled')
                 .settings-row
                     +number('Idle connection timeout:', `${communication}.idleConnectionTimeout`, '"idleConnectionTimeout"', 'true', '30000', '1',
                         'Maximum idle connection timeout upon which a connection to client will be closed')

From 0c72d825418f785a2b0017d4730b5f6e55f2f757 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Wed, 15 Mar 2017 11:43:23 +0700
Subject: [PATCH 055/516] IGNITE-4821 Implemented enforce join order option on
 query tab. (cherry picked from commit 94c1e7c)

---
 modules/web-console/backend/app/agent.js      | 14 ++++++-
 modules/web-console/backend/app/browser.js    |  8 ++--
 .../app/modules/agent/agent.module.js         | 10 +++--
 .../app/modules/sql/sql.controller.js         | 25 +++++++++---
 .../frontend/public/stylesheets/style.scss    |  8 +++-
 .../frontend/views/sql/sql.tpl.pug            | 39 ++++++++++++-------
 6 files changed, 74 insertions(+), 30 deletions(-)

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index 4cae8ee124175..b09e10a9c16c7 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -227,17 +227,27 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo,
          * @param {String} cacheName Cache name.
          * @param {String} query Query.
          * @param {Boolean} nonCollocatedJoins Flag whether to execute non collocated joins.
+         * @param {Boolean} enforceJoinOrder Flag whether enforce join order is enabled.
          * @param {Boolean} local Flag whether to execute query locally.
          * @param {int} pageSize Page size.
          * @returns {Promise}
          */
-        fieldsQuery(demo, nid, cacheName, query, nonCollocatedJoins, local, pageSize) {
+        fieldsQuery(demo, nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, local, pageSize) {
             const cmd = new Command(demo, 'exe')
                 .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
                 .addParam('p1', nid)
                 .addParam('p2', 'org.apache.ignite.internal.visor.query.VisorQueryTask');
 
-            if (nonCollocatedJoins) {
+            if (enforceJoinOrder) {
+                cmd.addParam('p3', 'org.apache.ignite.internal.visor.query.VisorQueryArgV3')
+                    .addParam('p4', cacheName)
+                    .addParam('p5', query)
+                    .addParam('p6', nonCollocatedJoins)
+                    .addParam('p7', enforceJoinOrder)
+                    .addParam('p8', local)
+                    .addParam('p9', pageSize);
+            }
+            else if (nonCollocatedJoins) {
                 cmd.addParam('p3', 'org.apache.ignite.internal.visor.query.VisorQueryArgV2')
                     .addParam('p4', cacheName)
                     .addParam('p5', query)
diff --git a/modules/web-console/backend/app/browser.js b/modules/web-console/backend/app/browser.js
index 2b1285e4f79c1..d272c70800435 100644
--- a/modules/web-console/backend/app/browser.js
+++ b/modules/web-console/backend/app/browser.js
@@ -98,9 +98,9 @@ module.exports.factory = (_, socketio, agentMgr, configure) => {
                 });
 
                 // Execute query on node and return first page to browser.
-                socket.on('node:query', (nid, cacheName, query, distributedJoins, local, pageSize, cb) => {
+                socket.on('node:query', (nid, cacheName, query, distributedJoins, enforceJoinOrder, local, pageSize, cb) => {
                     agentMgr.findAgent(accountId())
-                        .then((agent) => agent.fieldsQuery(demo, nid, cacheName, query, distributedJoins, local, pageSize))
+                        .then((agent) => agent.fieldsQuery(demo, nid, cacheName, query, distributedJoins, enforceJoinOrder, local, pageSize))
                         .then((res) => cb(null, res))
                         .catch((err) => cb(_errorToJson(err)));
                 });
@@ -114,13 +114,13 @@ module.exports.factory = (_, socketio, agentMgr, configure) => {
                 });
 
                 // Execute query on node and return full result to browser.
-                socket.on('node:query:getAll', (nid, cacheName, query, distributedJoins, local, cb) => {
+                socket.on('node:query:getAll', (nid, cacheName, query, distributedJoins, enforceJoinOrder, local, cb) => {
                     // Set page size for query.
                     const pageSize = 1024;
 
                     agentMgr.findAgent(accountId())
                         .then((agent) => {
-                            const firstPage = agent.fieldsQuery(demo, nid, cacheName, query, distributedJoins, local, pageSize)
+                            const firstPage = agent.fieldsQuery(demo, nid, cacheName, query, distributedJoins, enforceJoinOrder, local, pageSize)
                                 .then(({result}) => {
                                     if (result.key)
                                         return Promise.reject(result.key);
diff --git a/modules/web-console/frontend/app/modules/agent/agent.module.js b/modules/web-console/frontend/app/modules/agent/agent.module.js
index c0e92d5d2db99..724fc6c5d0914 100644
--- a/modules/web-console/frontend/app/modules/agent/agent.module.js
+++ b/modules/web-console/frontend/app/modules/agent/agent.module.js
@@ -265,12 +265,13 @@ class IgniteAgentMonitor {
      * @param {String} cacheName Cache name.
      * @param {String} [query] Query if null then scan query.
      * @param {Boolean} nonCollocatedJoins Flag whether to execute non collocated joins.
+     * @param {Boolean} enforceJoinOrder Flag whether enforce join order is enabled.
      * @param {Boolean} local Flag whether to execute query locally.
      * @param {int} pageSize
      * @returns {Promise}
      */
-    query(nid, cacheName, query, nonCollocatedJoins, local, pageSize) {
-        return this._rest('node:query', nid, maskNull(cacheName), maskNull(query), nonCollocatedJoins, local, pageSize)
+    query(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, local, pageSize) {
+        return this._rest('node:query', nid, maskNull(cacheName), maskNull(query), nonCollocatedJoins, enforceJoinOrder, local, pageSize)
             .then(({result}) => {
                 if (_.isEmpty(result.key))
                     return result.value;
@@ -284,11 +285,12 @@ class IgniteAgentMonitor {
      * @param {String} cacheName Cache name.
      * @param {String} [query] Query if null then scan query.
      * @param {Boolean} nonCollocatedJoins Flag whether to execute non collocated joins.
+     * @param {Boolean} enforceJoinOrder Flag whether enforce join order is enabled.
      * @param {Boolean} local Flag whether to execute query locally.
      * @returns {Promise}
      */
-    queryGetAll(nid, cacheName, query, nonCollocatedJoins, local) {
-        return this._rest('node:query:getAll', nid, maskNull(cacheName), maskNull(query), nonCollocatedJoins, local);
+    queryGetAll(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, local) {
+        return this._rest('node:query:getAll', nid, maskNull(cacheName), maskNull(query), nonCollocatedJoins, enforceJoinOrder, local);
     }
 
     /**
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index e2ead13ec7c41..2fd089d14f7a9 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -35,6 +35,8 @@ const SCAN_CACHE_WITH_FILTER_CASE_SENSITIVE = 'VISOR_SCAN_CACHE_WITH_FILTER_CASE
 
 const NON_COLLOCATED_JOINS_SINCE = '1.7.0';
 
+const ENFORCE_JOIN_VERS = [['1.7.9', '1.8.0'], ['1.8.4', '1.9.0'], ['1.9.1']];
+
 const _fullColName = (col) => {
     const res = [];
 
@@ -1337,7 +1339,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
             agentMonitor.awaitAgent()
                 .then(() => _closeOldQuery(paragraph))
                 .then(() => args.localNid || _chooseNode(args.cacheName, false))
-                .then((nid) => agentMonitor.query(nid, args.cacheName, args.query, args.nonCollocatedJoins, !!args.localNid, args.pageSize))
+                .then((nid) => agentMonitor.query(nid, args.cacheName, args.query, args.nonCollocatedJoins,
+                    args.enforceJoinOrder, !!args.localNid, args.pageSize))
                 .then(_processQueryResult.bind(this, paragraph, false))
                 .catch((err) => paragraph.errMsg = err.message);
         };
@@ -1370,8 +1373,18 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
             return false;
         };
 
+        $scope.enforceJoinOrderAvailable = (paragraph) => {
+            const cache = _.find($scope.caches, {name: paragraph.cacheName});
+
+            if (cache)
+                return !!_.find(cache.nodes, (node) => Version.includes(node.version, ...ENFORCE_JOIN_VERS));
+
+            return false;
+        };
+
         $scope.execute = (paragraph, local = false) => {
             const nonCollocatedJoins = !!paragraph.nonCollocatedJoins;
+            const enforceJoinOrder = !!paragraph.enforceJoinOrder;
 
             $scope.actionAvailable(paragraph, true) && _chooseNode(paragraph.cacheName, local)
                 .then((nid) => {
@@ -1391,6 +1404,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                                 pageSize: paragraph.pageSize,
                                 maxPages: paragraph.maxPages,
                                 nonCollocatedJoins,
+                                enforceJoinOrder,
                                 localNid: local ? nid : null
                             };
 
@@ -1398,7 +1412,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
                             ActivitiesData.post({ action: '/queries/execute' });
 
-                            return agentMonitor.query(nid, args.cacheName, qry, nonCollocatedJoins, local, args.pageSize);
+                            return agentMonitor.query(nid, args.cacheName, qry, nonCollocatedJoins, enforceJoinOrder, local, args.pageSize);
                         })
                         .then((res) => {
                             _processQueryResult(paragraph, true, res);
@@ -1451,7 +1465,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
                     ActivitiesData.post({ action: '/queries/explain' });
 
-                    return agentMonitor.query(nid, args.cacheName, args.query, false, false, args.pageSize);
+                    return agentMonitor.query(nid, args.cacheName, args.query, false, !!paragraph.enforceJoinOrder, false, args.pageSize);
                 })
                 .then(_processQueryResult.bind(this, paragraph, true))
                 .catch((err) => {
@@ -1489,7 +1503,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
                             ActivitiesData.post({ action: '/queries/scan' });
 
-                            return agentMonitor.query(nid, args.cacheName, query, false, local, args.pageSize);
+                            return agentMonitor.query(nid, args.cacheName, query, false, false, local, args.pageSize);
                         })
                         .then((res) => _processQueryResult(paragraph, true, res))
                         .catch((err) => {
@@ -1610,7 +1624,8 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
             const args = paragraph.queryArgs;
 
             return Promise.resolve(args.localNid || _chooseNode(args.cacheName, false))
-                .then((nid) => agentMonitor.queryGetAll(nid, args.cacheName, args.query, !!args.nonCollocatedJoins, !!args.localNid))
+                .then((nid) => agentMonitor.queryGetAll(nid, args.cacheName, args.query, !!args.nonCollocatedJoins,
+                    !!args.enforceJoinOrder, !!args.localNid))
                 .then((res) => _export(paragraph.name + '-all.csv', paragraph.gridOptions.columnDefs, res.columns, res.rows))
                 .catch(Messages.showError)
                 .then(() => paragraph.ace && paragraph.ace.focus());
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index ab7e3dd94286f..a07472e8840a4 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -809,7 +809,7 @@ button.form-control {
     }
 
     .btn-group {
-        vertical-align:top;
+        vertical-align: top;
         margin-left: 10px;
 
         i { line-height: 18px; }
@@ -1854,6 +1854,12 @@ th[st-sort] {
     display: inline-block;
 }
 
+.panel-top-align {
+    label {
+        vertical-align: top !important;
+    }
+}
+
 button.dropdown-toggle {
     margin-right: 5px;
 }
diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug
index 931d6327888d3..bc134a67e5bce 100644
--- a/modules/web-console/frontend/views/sql/sql.tpl.pug
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -91,20 +91,31 @@ mixin paragraph-rename
             input.form-control(id='paragraph-name-{{paragraph.id}}' ng-model='paragraph.editName' required ng-click='$event.stopPropagation();' ignite-on-enter='renameParagraph(paragraph, paragraph.editName)' ignite-on-escape='paragraph.edit = false')
 
 mixin query-settings
-    label.tipLabel(bs-tooltip data-placement='bottom' data-title='Configure periodical execution of last successfully executed query') Refresh rate:
-        button.btn.btn-default.fa.fa-clock-o.tipLabel(ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='{{ $ctrl.paragraphRateTemplateUrl }}' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
-
-    label.tipLabel(bs-tooltip data-placement='bottom' data-title='Max number of rows to show in query result as one page') Page size:
-        button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-select bs-options='item for item in pageSizes')
-
-    label.tipLabel(bs-tooltip data-placement='bottom' data-title='Limit query max results to specified number of pages') Max pages:
-        button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.maxPages' bs-select bs-options='item.value as item.label for item in maxPages')
-
-    label.tipLabel(ng-if='nonCollocatedJoinsAvailable(paragraph)' bs-tooltip data-placement='bottom' data-title='Non-collocated joins is a special mode that allow to join data across cluster without collocation.<br/>\
-        Nested joins are not supported for now.<br/>\
-        <b>NOTE</b>: In some cases it may consume more heap memory or may take a long time than collocated joins.' data-trigger='hover')
-        input(type='checkbox' ng-model='paragraph.nonCollocatedJoins')
-        span Allow non-collocated joins
+    .panel-top-align
+        label.tipLabel(bs-tooltip data-placement='bottom' data-title='Configure periodical execution of last successfully executed query') Refresh rate:
+            button.btn.btn-default.fa.fa-clock-o.tipLabel(ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='{{ $ctrl.paragraphRateTemplateUrl }}' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
+
+        label.tipLabel(bs-tooltip data-placement='bottom' data-title='Max number of rows to show in query result as one page') Page size:
+            button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-select bs-options='item for item in pageSizes')
+
+        label.tipLabel(bs-tooltip data-placement='bottom' data-title='Limit query max results to specified number of pages') Max pages:
+            button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.maxPages' bs-select bs-options='item.value as item.label for item in maxPages')
+
+        .panel-tip-container
+            .row(ng-if='nonCollocatedJoinsAvailable(paragraph)')
+                label.tipLabel(bs-tooltip data-placement='bottom' data-title='Non-collocated joins is a special mode that allow to join data across cluster without collocation.<br/>\
+                    Nested joins are not supported for now.<br/>\
+                    <b>NOTE</b>: In some cases it may consume more heap memory or may take a long time than collocated joins.' data-trigger='hover')
+                    input(type='checkbox' ng-model='paragraph.nonCollocatedJoins')
+                    span Allow non-collocated joins
+            .row(ng-if='enforceJoinOrderAvailable(paragraph)')
+                label.tipLabel(bs-tooltip data-placement='bottom' data-title='Enforce join order of tables in the query.<br/>\
+                    If <b>set</b> then query optimizer will not reorder tables in join.<br/>\
+                    <b>NOTE:</b> It is not recommended to enable this property until you are sure that\
+                    your indexes and the query itself are correct and tuned as much as possible but\
+                    query optimizer still produces wrong join order.' data-trigger='hover')
+                    input(type='checkbox' ng-model='paragraph.enforceJoinOrder')
+                    span Enforce join order
 
 mixin query-actions
     button.btn.btn-primary(ng-disabled='!actionAvailable(paragraph, true)' ng-click='execute(paragraph)') Execute

From baf8a8ca01d7e2fc69b04a77a39f7ff1f8b35727 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 17 Feb 2017 18:02:48 +0700
Subject: [PATCH 056/516] IGNITE-4472 Minor fix. (cherry picked from commit
 7cb3e68)

---
 .../backend/services/activities.js            |  7 ++---
 .../list-of-registered-users.controller.js    | 14 ++++------
 .../configuration/configuration.module.js     |  6 ++--
 .../configuration/summary/summary.worker.js   | 28 +++++++++----------
 .../console/agent/handlers/RestListener.java  |  9 +++++-
 5 files changed, 33 insertions(+), 31 deletions(-)

diff --git a/modules/web-console/backend/services/activities.js b/modules/web-console/backend/services/activities.js
index 124c775dc7a86..afde8e72f3ad3 100644
--- a/modules/web-console/backend/services/activities.js
+++ b/modules/web-console/backend/services/activities.js
@@ -37,10 +37,10 @@ module.exports.factory = (_, mongo) => {
          * @param {String} owner - User ID
          * @param {String} action - Action string presentation.
          * @param {String} group - Action group string presentation.
-         * @param {Date} [date] - Optional date to save in activity.
+         * @param {Date} [now] - Optional date to save in activity.
          * @returns {Promise.<mongo.ObjectId>} that resolve activity
          */
-        static merge(owner, {action, group}, date = new Date()) {
+        static merge(owner, {action, group}, now = new Date()) {
             mongo.Account.findById(owner)
                 .then((user) => {
                     user.lastActivity = new Date();
@@ -48,8 +48,7 @@ module.exports.factory = (_, mongo) => {
                     return user.save();
                 });
 
-            date.setDate(1);
-            date.setHours(0, 0, 0, 0);
+            const date = Date.UTC(now.getFullYear(), now.getMonth(), 1);
 
             return mongo.Activities.findOne({owner, action, date}).exec()
                 .then((activity) => {
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 33fe819068c78..54971b142b014 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -44,9 +44,6 @@ export default class IgniteListOfRegisteredUsersCtrl {
             startDate: new Date()
         };
 
-        $ctrl.params.startDate.setDate(1);
-        $ctrl.params.startDate.setHours(0, 0, 0, 0);
-
         const columnCompany = _.find(columnDefs, { displayName: 'Company' });
         const columnCountry = _.find(columnDefs, { displayName: 'Country' });
 
@@ -151,7 +148,7 @@ export default class IgniteListOfRegisteredUsersCtrl {
         };
 
         /**
-         * @param {{startDate: Date, endDate: Date}} params
+         * @param {{startDate: number, endDate: number}} params
          */
         const reloadUsers = (params) => {
             AdminData.loadUsers(params)
@@ -177,11 +174,10 @@ export default class IgniteListOfRegisteredUsersCtrl {
         $scope.$watch(() => $ctrl.params.startDate, (dt) => {
             $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(dt, 'yyyy_MM')}.csv`;
 
-            const endDate = new Date(dt);
-
-            endDate.setMonth(endDate.getMonth() + 1);
+            const startDate = Date.UTC(dt.getFullYear(), dt.getMonth(), 1);
+            const endDate = Date.UTC(dt.getFullYear(), dt.getMonth() + 1, 1);
 
-            reloadUsers({startDate: dtFilter(dt, 'yyyy-MM-dd'), endDate: dtFilter(endDate, 'yyyy-MM-dd')});
+            reloadUsers({ startDate, endDate });
         });
     }
 
@@ -237,6 +233,6 @@ export default class IgniteListOfRegisteredUsersCtrl {
     }
 
     exportCsv() {
-        this.gridApi.exporter.csvExport('all', 'visible');
+        this.gridApi.exporter.csvExport('visible', 'visible');
     }
 }
diff --git a/modules/web-console/frontend/app/modules/configuration/configuration.module.js b/modules/web-console/frontend/app/modules/configuration/configuration.module.js
index 4288ff79ea182..51306c6d8f8c5 100644
--- a/modules/web-console/frontend/app/modules/configuration/configuration.module.js
+++ b/modules/web-console/frontend/app/modules/configuration/configuration.module.js
@@ -49,10 +49,10 @@ angular
 ])
 .provider(...igniteSidebar)
 .directive(...igniteSidebarDirective)
-.service('IgniteConfigurationGenerator', IgniteConfigurationGenerator)
+.service('IgniteConfigurationGenerator', () => IgniteConfigurationGenerator)
 .service('IgnitePlatformGenerator', IgnitePlatformGenerator)
-.service('SpringTransformer', IgniteSpringTransformer)
-.service('JavaTransformer', IgniteJavaTransformer)
+.service('SpringTransformer', () => IgniteSpringTransformer)
+.service('JavaTransformer', () => IgniteJavaTransformer)
 .service('IgniteSharpTransformer', SharpTransformer)
 .service('IgniteVersion', IgniteVersion)
 .service('IgniteEventGroups', IgniteEventGroups)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
index 070b6ce87aa69..7ea1d5ad5beb0 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
@@ -17,24 +17,24 @@
 
 import JSZip from 'jszip';
 
-import MavenGenerator from 'app/modules/configuration/generator/Maven.service';
-import DockerGenerator from 'app/modules/configuration/generator/Docker.service';
-import ReadmeGenerator from 'app/modules/configuration/generator/Readme.service';
-import PropertiesGenerator from 'app/modules/configuration/generator/Properties.service';
-import ConfigurationGenerator from 'app/modules/configuration/generator/ConfigurationGenerator';
+import IgniteMavenGenerator from 'app/modules/configuration/generator/Maven.service';
+import IgniteDockerGenerator from 'app/modules/configuration/generator/Docker.service';
+import IgniteReadmeGenerator from 'app/modules/configuration/generator/Readme.service';
+import IgnitePropertiesGenerator from 'app/modules/configuration/generator/Properties.service';
+import IgniteConfigurationGenerator from 'app/modules/configuration/generator/ConfigurationGenerator';
 
-import JavaTransformer from 'app/modules/configuration/generator/JavaTransformer.service';
-import SpringTransformer from 'app/modules/configuration/generator/SpringTransformer.service';
+import IgniteJavaTransformer from 'app/modules/configuration/generator/JavaTransformer.service';
+import IgniteSpringTransformer from 'app/modules/configuration/generator/SpringTransformer.service';
 
-const maven = new MavenGenerator();
-const docker = new DockerGenerator();
-const readme = new ReadmeGenerator();
-const properties = new PropertiesGenerator();
+const maven = new IgniteMavenGenerator();
+const docker = new IgniteDockerGenerator();
+const readme = new IgniteReadmeGenerator();
+const properties = new IgnitePropertiesGenerator();
 
-const java = new JavaTransformer[0]();
-const spring = new SpringTransformer[0]();
+const java = IgniteJavaTransformer;
+const spring = IgniteSpringTransformer;
 
-const generator = new ConfigurationGenerator[0]();
+const generator = IgniteConfigurationGenerator;
 
 const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
 
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
index 1e86549e25927..fcacc88182bcc 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
@@ -38,6 +38,7 @@
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.ignite.console.agent.AgentConfiguration;
 import org.apache.ignite.console.demo.AgentClusterDemo;
 import org.apache.log4j.Logger;
@@ -65,7 +66,13 @@ public RestListener(AgentConfiguration cfg) {
 
         this.cfg = cfg;
 
-        httpClient = HttpClientBuilder.create().build();
+        // Create a connection manager with custom configuration.
+        PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager();
+
+        connMgr.setDefaultMaxPerRoute(Integer.MAX_VALUE);
+        connMgr.setMaxTotal(Integer.MAX_VALUE);
+
+        httpClient = HttpClientBuilder.create().setConnectionManager(connMgr).build();
     }
 
     /** {@inheritDoc} */

From 4ba2d12c1a797bc9cc5ce585354adfe3755a3ddb Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 17 Mar 2017 11:08:39 +0700
Subject: [PATCH 057/516] IGNITE-4666 Clone to Input service. (cherry picked
 from commit 83579ce)

---
 modules/web-console/frontend/app/app.js       | 10 +--
 .../app/components/input-dialog/index.js      | 24 +++++
 .../input-dialog/input-dialog.controller.js   | 35 ++++++++
 .../input-dialog/input-dialog.service.js      | 88 +++++++++++++++++++
 .../input-dialog/input-dialog.tpl.pug}        | 10 +--
 .../frontend/app/services/Clone.service.js    | 66 --------------
 .../frontend/controllers/caches-controller.js | 10 +--
 .../controllers/clusters-controller.js        |  6 +-
 .../controllers/domains-controller.js         | 12 ++-
 .../frontend/controllers/igfs-controller.js   | 10 +--
 10 files changed, 173 insertions(+), 98 deletions(-)
 create mode 100644 modules/web-console/frontend/app/components/input-dialog/index.js
 create mode 100644 modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js
 create mode 100644 modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js
 rename modules/web-console/frontend/{views/templates/clone.tpl.pug => app/components/input-dialog/input-dialog.tpl.pug} (83%)
 delete mode 100644 modules/web-console/frontend/app/services/Clone.service.js

diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 5e3bb07434b2c..1e21d247c060a 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -16,9 +16,9 @@
  */
 
 import '../public/stylesheets/style.scss';
-import '../app/components/ui-grid-header/ui-grid-header.scss';
-import '../app/components/ui-grid-settings/ui-grid-settings.scss';
-import '../app/components/form-field-datepicker/form-field-datepicker.scss';
+import './components/ui-grid-header/ui-grid-header.scss';
+import './components/ui-grid-settings/ui-grid-settings.scss';
+import './components/form-field-datepicker/form-field-datepicker.scss';
 
 import './app.config';
 
@@ -80,7 +80,6 @@ import igniteRetainSelection from './directives/retain-selection.directive';
 
 // Services.
 import ChartColors from './services/ChartColors.service';
-import Clone from './services/Clone.service.js';
 import Confirm from './services/Confirm.service.js';
 import ConfirmBatch from './services/ConfirmBatch.service.js';
 import CopyToClipboard from './services/CopyToClipboard.service';
@@ -117,6 +116,7 @@ import resetPassword from './controllers/reset-password.controller';
 // Components
 import igniteListOfRegisteredUsers from './components/list-of-registered-users';
 import IgniteActivitiesUserDialog from './components/activities-user-dialog';
+import './components/input-dialog';
 
 // Inject external modules.
 import 'ignite_modules_temp/index';
@@ -149,6 +149,7 @@ angular
     'ignite-console.core',
     'ignite-console.ace',
     'ignite-console.Form',
+    'ignite-console.input-dialog',
     'ignite-console.user',
     'ignite-console.branding',
     'ignite-console.socket',
@@ -203,7 +204,6 @@ angular
 .service('JavaTypes', JavaTypes)
 .service('SqlTypes', SqlTypes)
 .service(...ChartColors)
-.service(...Clone)
 .service(...Confirm)
 .service(...ConfirmBatch)
 .service(...CopyToClipboard)
diff --git a/modules/web-console/frontend/app/components/input-dialog/index.js b/modules/web-console/frontend/app/components/input-dialog/index.js
new file mode 100644
index 0000000000000..4bb96422160b4
--- /dev/null
+++ b/modules/web-console/frontend/app/components/input-dialog/index.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import inputDialog from './input-dialog.service';
+
+angular
+    .module('ignite-console.input-dialog', [])
+    .service('IgniteInput', inputDialog);
diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js b/modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js
new file mode 100644
index 0000000000000..3f6e97b96c9a6
--- /dev/null
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class InputDialogController {
+    static $inject = ['deferred', 'ui'];
+
+    constructor(deferred, {title, label, value, toValidValue}) {
+        this.deferred = deferred;
+        this.title = title;
+        this.label = label;
+        this.value = value;
+        this.toValidValue = toValidValue;
+    }
+
+    confirm() {
+        if (_.isFunction(this.toValidValue))
+            return this.deferred.resolve(this.toValidValue(this.value));
+
+        this.deferred.resolve(this.value);
+    }
+}
diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js
new file mode 100644
index 0000000000000..fc3cb851c720f
--- /dev/null
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import controller from './input-dialog.controller';
+import templateUrl from './input-dialog.tpl.pug';
+
+export default class InputDialog {
+    static $inject = ['$modal', '$q'];
+
+    constructor($modal, $q) {
+        this.$modal = $modal;
+        this.$q = $q;
+    }
+
+    /**
+     * Open input dialog to configure custom value.
+     *
+     * @param {String} title Dialog title.
+     * @param {String} label Input field label.
+     * @param {String} value Default value.
+     * @param {Function} [toValidValue] Validator function.
+     * @returns {Promise.<String>} User input.
+     */
+    input(title, label, value, toValidValue) {
+        const deferred = this.$q.defer();
+
+        const modal = this.$modal({
+            templateUrl,
+            resolve: {
+                deferred: () => deferred,
+                ui: () => ({
+                    title,
+                    label,
+                    value,
+                    toValidValue
+                })
+            },
+            placement: 'center',
+            controller,
+            controllerAs: 'ctrl'
+        });
+
+        const modalHide = modal.hide;
+
+        modal.hide = () => deferred.reject('cancelled');
+
+        return deferred.promise
+            .finally(modalHide);
+    }
+
+    /**
+     * Open input dialog to configure cloned object name.
+     *
+     * @param {String} srcName Name of source object.
+     * @param {Array.<String>} names List of already exist names.
+     * @returns {Promise.<String>} New name
+     */
+    clone(srcName, names) {
+        const uniqueName = (value) => {
+            let num = 1;
+            let tmpName = value;
+
+            while (_.includes(names, tmpName)) {
+                tmpName = `${value}_${num}`;
+
+                num++;
+            }
+
+            return tmpName;
+        };
+
+        return this.input('Clone', 'New name', uniqueName(srcName), uniqueName);
+    }
+}
diff --git a/modules/web-console/frontend/views/templates/clone.tpl.pug b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
similarity index 83%
rename from modules/web-console/frontend/views/templates/clone.tpl.pug
rename to modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
index e62cb18a8fb60..95549d7995e21 100644
--- a/modules/web-console/frontend/views/templates/clone.tpl.pug
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
@@ -23,17 +23,17 @@ include /app/helpers/jade/mixins
                 button.close(ng-click='$hide()') &times;
                 h4.modal-title 
                     i.fa.fa-clone
-                    | Clone
+                    | {{ ctrl.title }}
             form.form-horizontal.modal-body.row(name='ui.inputForm' novalidate)
                 div
                     .col-sm-2
-                        label.required.labelFormField New name:&nbsp;
+                        label.required.labelFormField {{ ctrl.label }}:&nbsp;
                     .col-sm-10
                         .input-tip
-                            +ignite-form-field-input('"copy-new-name"','newName', false, 'true', 'Enter new name')(
+                            +ignite-form-field-input('"input-field"', 'ctrl.value', false, 'true', 'Enter value')(
                                 data-ignite-form-field-input-autofocus='true'
-                                ignite-on-enter='form.$valid && ok(newName)'
+                                ignite-on-enter='form.$valid && ctrl.confirm()'
                             )
             .modal-footer
                 button.btn.btn-default(id='copy-btn-cancel' ng-click='$hide()') Cancel
-                button.btn.btn-primary(id='copy-btn-confirm' ng-disabled='ui.inputForm.$invalid' ng-click='ok(newName)') Confirm
+                button.btn.btn-primary(id='copy-btn-confirm' ng-disabled='ui.inputForm.$invalid' ng-click='ctrl.confirm()') Confirm
diff --git a/modules/web-console/frontend/app/services/Clone.service.js b/modules/web-console/frontend/app/services/Clone.service.js
deleted file mode 100644
index d079141364b55..0000000000000
--- a/modules/web-console/frontend/app/services/Clone.service.js
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import templateUrl from 'views/templates/clone.tpl.pug';
-
-// Service for clone objects.
-export default ['IgniteClone', ['$rootScope', '$q', '$modal', ($root, $q, $modal) => {
-    const scope = $root.$new();
-
-    let _names = [];
-    let deferred;
-    let _validator;
-
-    function _nextAvailableName(name) {
-        let num = 1;
-        let tmpName = name;
-
-        while (_.includes(_names, tmpName)) {
-            tmpName = name + '_' + num.toString();
-
-            num++;
-        }
-
-        return tmpName;
-    }
-
-    const cloneModal = $modal({templateUrl, scope, placement: 'center', show: false});
-
-    scope.ok = function(newName) {
-        if (!_validator || _validator(newName)) {
-            deferred.resolve(_nextAvailableName(newName));
-
-            cloneModal.hide();
-        }
-    };
-
-    cloneModal.confirm = function(oldName, names, validator) {
-        _names = names;
-
-        scope.newName = _nextAvailableName(oldName);
-
-        _validator = validator;
-
-        deferred = $q.defer();
-
-        cloneModal.$promise.then(cloneModal.show);
-
-        return deferred.promise;
-    };
-
-    return cloneModal;
-}]];
diff --git a/modules/web-console/frontend/controllers/caches-controller.js b/modules/web-console/frontend/controllers/caches-controller.js
index b50fde3188b1a..d4a13e56c74a2 100644
--- a/modules/web-console/frontend/controllers/caches-controller.js
+++ b/modules/web-console/frontend/controllers/caches-controller.js
@@ -19,8 +19,8 @@ import infoMessageTemplateUrl from 'views/templates/message.tpl.pug';
 
 // Controller for Caches screen.
 export default ['cachesController', [
-    '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'IgniteLegacyTable',
-    function($scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Confirm, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, Resource, ErrorPopover, FormUtils, LegacyTable) {
+    '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteInput', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'IgniteLegacyTable',
+    function($scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Confirm, Input, Loading, ModelNormalizer, UnsavedChangesGuard, Resource, ErrorPopover, FormUtils, LegacyTable) {
         UnsavedChangesGuard.install($scope);
 
         const emptyCache = {empty: true};
@@ -517,15 +517,13 @@ export default ['cachesController', [
         };
 
         function _cacheNames() {
-            return _.map($scope.caches, function(cache) {
-                return cache.name;
-            });
+            return _.map($scope.caches, (cache) => cache.name);
         }
 
         // Clone cache with new name.
         $scope.cloneItem = function() {
             if (validate($scope.backupItem)) {
-                Clone.confirm($scope.backupItem.name, _cacheNames()).then(function(newName) {
+                Input.clone($scope.backupItem.name, _cacheNames()).then((newName) => {
                     const item = angular.copy($scope.backupItem);
 
                     delete item._id;
diff --git a/modules/web-console/frontend/controllers/clusters-controller.js b/modules/web-console/frontend/controllers/clusters-controller.js
index 7f90b90e198fb..c8392cf7ef0aa 100644
--- a/modules/web-console/frontend/controllers/clusters-controller.js
+++ b/modules/web-console/frontend/controllers/clusters-controller.js
@@ -17,8 +17,8 @@
 
 // Controller for Clusters screen.
 export default ['clustersController', [
-    '$rootScope', '$scope', '$http', '$state', '$timeout', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteEventGroups', 'DemoInfo', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils',
-    function($root, $scope, $http, $state, $timeout, LegacyUtils, Messages, Confirm, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, igniteEventGroups, DemoInfo, LegacyTable, Resource, ErrorPopover, FormUtils) {
+    '$rootScope', '$scope', '$http', '$state', '$timeout', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteInput', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteEventGroups', 'DemoInfo', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils',
+    function($root, $scope, $http, $state, $timeout, LegacyUtils, Messages, Confirm, Input, Loading, ModelNormalizer, UnsavedChangesGuard, igniteEventGroups, DemoInfo, LegacyTable, Resource, ErrorPopover, FormUtils) {
         UnsavedChangesGuard.install($scope);
 
         const emptyCluster = {empty: true};
@@ -774,7 +774,7 @@ export default ['clustersController', [
         // Clone cluster with new name.
         $scope.cloneItem = function() {
             if (validate($scope.backupItem)) {
-                Clone.confirm($scope.backupItem.name, _clusterNames()).then(function(newName) {
+                Input.clone($scope.backupItem.name, _clusterNames()).then((newName) => {
                     const item = angular.copy($scope.backupItem);
 
                     delete item._id;
diff --git a/modules/web-console/frontend/controllers/domains-controller.js b/modules/web-console/frontend/controllers/domains-controller.js
index 5c9e511a139ca..806dd4577ead9 100644
--- a/modules/web-console/frontend/controllers/domains-controller.js
+++ b/modules/web-console/frontend/controllers/domains-controller.js
@@ -19,8 +19,8 @@ import templateUrl from 'views/configuration/domains-import.tpl.pug';
 
 // Controller for Domain model screen.
 export default ['domainsController', [
-    '$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteAgentMonitor', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes', 'IgniteActivitiesData',
-    function($root, $scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Focus, Confirm, ConfirmBatch, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, IgniteAgentMonitor, LegacyTable, Resource, ErrorPopover, FormUtils, JavaTypes, SqlTypes, ActivitiesData) {
+    '$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteInput', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteAgentMonitor', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes', 'IgniteActivitiesData',
+    function($root, $scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Focus, Confirm, ConfirmBatch, Input, Loading, ModelNormalizer, UnsavedChangesGuard, IgniteAgentMonitor, LegacyTable, Resource, ErrorPopover, FormUtils, JavaTypes, SqlTypes, ActivitiesData) {
         UnsavedChangesGuard.install($scope);
 
         const emptyDomain = {empty: true};
@@ -1429,7 +1429,7 @@ export default ['domainsController', [
 
                 item.cacheStoreChanges = [];
 
-                _.forEach(item.caches, function(cacheId) {
+                _.forEach(item.caches, (cacheId) => {
                     const cache = _.find($scope.caches, {value: cacheId}).cache;
 
                     const change = LegacyUtils.autoCacheStoreConfiguration(cache, [item]);
@@ -1444,9 +1444,7 @@ export default ['domainsController', [
         };
 
         function _domainNames() {
-            return _.map($scope.domains, function(domain) {
-                return domain.valueType;
-            });
+            return _.map($scope.domains, (domain) => domain.valueType);
         }
 
         function _newNameIsValidJavaClass(newName) {
@@ -1457,7 +1455,7 @@ export default ['domainsController', [
         // Save domain model with new name.
         $scope.cloneItem = function() {
             if ($scope.tableReset(true) && validate($scope.backupItem)) {
-                Clone.confirm($scope.backupItem.valueType, _domainNames(), _newNameIsValidJavaClass).then(function(newName) {
+                Input.clone($scope.backupItem.valueType, _domainNames(), _newNameIsValidJavaClass).then((newName) => {
                     const item = angular.copy($scope.backupItem);
 
                     delete item._id;
diff --git a/modules/web-console/frontend/controllers/igfs-controller.js b/modules/web-console/frontend/controllers/igfs-controller.js
index b3c6043ce32b6..504e28d96e417 100644
--- a/modules/web-console/frontend/controllers/igfs-controller.js
+++ b/modules/web-console/frontend/controllers/igfs-controller.js
@@ -17,8 +17,8 @@
 
 // Controller for IGFS screen.
 export default ['igfsController', [
-    '$scope', '$http', '$state', '$filter', '$timeout', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteClone', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils',
-    function($scope, $http, $state, $filter, $timeout, LegacyUtils, Messages, Confirm, Clone, Loading, ModelNormalizer, UnsavedChangesGuard, LegacyTable, Resource, ErrorPopover, FormUtils) {
+    '$scope', '$http', '$state', '$filter', '$timeout', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteInput', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils',
+    function($scope, $http, $state, $filter, $timeout, LegacyUtils, Messages, Confirm, Input, Loading, ModelNormalizer, UnsavedChangesGuard, LegacyTable, Resource, ErrorPopover, FormUtils) {
         UnsavedChangesGuard.install($scope);
 
         const emptyIgfs = {empty: true};
@@ -328,15 +328,13 @@ export default ['igfsController', [
         };
 
         function _igfsNames() {
-            return _.map($scope.igfss, function(igfs) {
-                return igfs.name;
-            });
+            return _.map($scope.igfss, (igfs) => igfs.name);
         }
 
         // Clone IGFS with new name.
         $scope.cloneItem = function() {
             if ($scope.tableReset(true) && validate($scope.backupItem)) {
-                Clone.confirm($scope.backupItem.name, _igfsNames()).then(function(newName) {
+                Input.clone($scope.backupItem.name, _igfsNames()).then((newName) => {
                     const item = angular.copy($scope.backupItem);
 
                     delete item._id;

From f1aa86f43776840ff14d5c709f3599da2e7b39e5 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Tue, 14 Mar 2017 15:54:14 +0700
Subject: [PATCH 058/516] Web Console: Cleanup (cherry picked from commit
 3da7794)

---
 .../ui-grid-settings/ui-grid-settings.scss    | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
index 4beb2a1baa40a..24f4d9b7efb7f 100644
--- a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
@@ -70,6 +70,41 @@
         }
     }
 
+    &-number-filter {
+        float: right;
+
+        .ignite-form-field {
+            width: 180px;
+            margin-right: 0;
+
+            &__label {
+            }
+
+            &__control {
+            }
+
+            &:nth-child(1) {
+                float: left;
+
+                .ignite-form-field__label {
+                    margin-right: 0;
+                    width: 70%;
+                    max-width: 100%;
+                }
+
+                .ignite-form-field__control {
+                    width: 30%;
+                }
+            }
+        }
+
+        button {
+            width: auto;
+            display: inline-block;
+            margin-left: 5px;
+        }
+    }
+
     &-dateperiod {
         float: right;
 

From d0c0042d3fbdade16e719cfb91acebae3f10bf04 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 2 Mar 2017 16:36:43 +0700
Subject: [PATCH 059/516] Minor fixes. (cherry picked from commit 684dc7b)

---
 modules/web-console/backend/app/agent.js      | 36 +++++++++++++++++++
 modules/web-console/backend/app/browser.js    | 28 +++++++++++++++
 .../app/modules/sql/sql.controller.js         |  1 -
 .../demo/service/DemoCachesLoadService.java   | 26 ++++++++++++++
 4 files changed, 90 insertions(+), 1 deletion(-)

diff --git a/modules/web-console/backend/app/agent.js b/modules/web-console/backend/app/agent.js
index b09e10a9c16c7..f65eabb29bee7 100644
--- a/modules/web-console/backend/app/agent.js
+++ b/modules/web-console/backend/app/agent.js
@@ -338,6 +338,42 @@ module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo,
             return this.executeRest(cmd);
         }
 
+        /**
+         * Collect running queries
+         * @param {Boolean} demo Is need run command on demo node.
+         * @param {Number} duration minimum duration time of running queries.
+         * @returns {Promise}
+         */
+        queryCollectRunning(demo, duration) {
+            const cmd = new Command(demo, 'exe')
+                .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
+                .addParam('p1', '')
+                .addParam('p2', 'org.apache.ignite.internal.visor.query.VisorCollectRunningQueriesTask')
+                .addParam('p3', 'java.lang.Long')
+                .addParam('p4', duration);
+
+            return this.executeRest(cmd);
+        }
+
+        /**
+         * Cancel running query.
+         * @param {Boolean} demo Is need run command on demo node.
+         * @param {String} nid Node id.
+         * @param {Number} queryId query id to cancel.
+         * @returns {Promise}
+         */
+        queryCancel(demo, nid, queryId) {
+            const cmd = new Command(demo, 'exe')
+                .addParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
+                .addParam('p1', nid)
+                .addParam('p2', 'org.apache.ignite.internal.visor.query.VisorCancelQueriesTask')
+                .addParam('p3', 'java.util.Collection')
+                .addParam('p4', 'java.lang.Long')
+                .addParam('p5', queryId);
+
+            return this.executeRest(cmd);
+        }
+
         /**
          * @param {Boolean} demo Is need run command on demo node.
          * @param {String} cacheName Cache name.
diff --git a/modules/web-console/backend/app/browser.js b/modules/web-console/backend/app/browser.js
index d272c70800435..00ae751f93dae 100644
--- a/modules/web-console/backend/app/browser.js
+++ b/modules/web-console/backend/app/browser.js
@@ -175,6 +175,34 @@ module.exports.factory = (_, socketio, agentMgr, configure) => {
                         .catch((err) => cb(_errorToJson(err)));
                 });
 
+                // Collect running queries from all nodes in grid.
+                socket.on('node:query:running', (duration, cb) => {
+                    agentMgr.findAgent(accountId())
+                        .then((agent) => agent.queryCollectRunning(demo, duration))
+                        .then((data) => {
+
+                            if (data.finished)
+                                return cb(null, data.result);
+
+                            cb(_errorToJson(data.error));
+                        })
+                        .catch((err) => cb(_errorToJson(err)));
+                });
+
+                // Cancel running query by query id on node.
+                socket.on('node:query:cancel', (nid, queryId, cb) => {
+                    agentMgr.findAgent(accountId())
+                        .then((agent) => agent.queryCancel(demo, nid, queryId))
+                        .then((data) => {
+
+                            if (data.finished)
+                                return cb(null, data.result);
+
+                            cb(_errorToJson(data.error));
+                        })
+                        .catch((err) => cb(_errorToJson(err)));
+                });
+
                 // Return cache metadata from all nodes in grid.
                 socket.on('node:cache:metadata', (cacheName, cb) => {
                     agentMgr.findAgent(accountId())
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 2fd089d14f7a9..7ded2d5d99b6f 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -20,7 +20,6 @@ import cacheMetadataTemplateUrl from 'views/sql/cache-metadata.tpl.pug';
 import chartSettingsTemplateUrl from 'views/sql/chart-settings.tpl.pug';
 import showQueryTemplateUrl from 'views/templates/message.tpl.pug';
 
-
 // Time line X axis descriptor.
 const TIME_LINE = {value: -1, type: 'java.sql.Date', label: 'TIME_LINE'};
 
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
index fbfa2ae6e09e2..5f7823b418e1d 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.cache.QueryIndex;
 import org.apache.ignite.cache.QueryIndexType;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.console.demo.AgentDemoUtils;
 import org.apache.ignite.console.demo.model.Car;
@@ -209,6 +210,7 @@ private static <K, V> CacheConfiguration<K, V> cacheConfiguration(String name) {
         ccfg.setQueryDetailMetricsSize(10);
         ccfg.setStartSize(100);
         ccfg.setStatisticsEnabled(true);
+        ccfg.setSqlFunctionClasses(SQLFunctions.class);
 
         return ccfg;
     }
@@ -453,4 +455,28 @@ private void populateCacheCar() {
         if (ignite.log().isDebugEnabled())
             ignite.log().debug("DEMO: Finished cars population.");
     }
+
+    /**
+     * Utility class with custom SQL functions.
+     */
+    public static class SQLFunctions {
+        /**
+         * Sleep function to simulate long running queries.
+         *
+         * @param x Time to sleep.
+         * @return Return specified argument.
+         */
+        @QuerySqlFunction
+        public static long sleep(long x) {
+            if (x >= 0)
+                try {
+                    Thread.sleep(x);
+                }
+                catch (InterruptedException ignored) {
+                    // No-op.
+                }
+
+            return x;
+        }
+    }
 }

From d939944f84aa415099a945b720dd4cd65132f6af Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 21 Mar 2017 10:05:33 +0700
Subject: [PATCH 060/516] IGNITE-1.8.4 Manual merge with master.

---
 .../frontend/gulpfile.babel.js/tasks/jade.js  | 49 -----------------
 .../webpack/environments/test.js              | 52 ++++++++-----------
 2 files changed, 22 insertions(+), 79 deletions(-)
 delete mode 100644 modules/web-console/frontend/gulpfile.babel.js/tasks/jade.js

diff --git a/modules/web-console/frontend/gulpfile.babel.js/tasks/jade.js b/modules/web-console/frontend/gulpfile.babel.js/tasks/jade.js
deleted file mode 100644
index 49359764bdeac..0000000000000
--- a/modules/web-console/frontend/gulpfile.babel.js/tasks/jade.js
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import gulp from 'gulp';
-import ll from 'gulp-ll';
-import jade from 'gulp-jade';
-
-import { jadeViewsPaths, jadeAppModulePaths, jadeModulePaths, destDir, rootDir } from '../paths';
-
-const jadeOptions = {
-    basedir: rootDir,
-    cache: true
-};
-
-ll.tasks(['jade:views', 'jade:app', 'jade:ignite_modules']);
-
-gulp.task('jade', ['jade:views', 'jade:app', 'jade:ignite_modules']);
-
-gulp.task('jade:views', () =>
-    gulp.src(jadeViewsPaths)
-        .pipe(jade(jadeOptions))
-        .pipe(gulp.dest(destDir))
-);
-
-gulp.task('jade:app', () =>
-    gulp.src(jadeAppModulePaths)
-        .pipe(jade(jadeOptions))
-        .pipe(gulp.dest(destDir))
-);
-
-gulp.task('jade:ignite_modules', () =>
-    gulp.src(jadeModulePaths)
-        .pipe(jade(jadeOptions))
-        .pipe(gulp.dest(`${destDir}/ignite_modules`))
-);
diff --git a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/test.js b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/test.js
index 1c37196838ae4..0cb9b77980840 100644
--- a/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/test.js
+++ b/modules/web-console/frontend/gulpfile.babel.js/webpack/environments/test.js
@@ -19,34 +19,26 @@ import webpack from 'webpack';
 
 const NODE_ENV = process.env.NODE_ENV || 'production';
 
-export default () => {
-
-    return {
-        cache: true,
-        node: {
-            fs: 'empty'
-        },
-
-        module: {
-            preLoaders: null
-        },
-
-        // Entry points.
-        entry: null,
-
-        // Output system.
-        output: null,
-        eslint: null,
-
-        // Load plugins.
-        plugins: [
-            new webpack.ProvidePlugin({
-                $: 'jquery',
-                jQuery: 'jquery',
-                _: 'lodash',
-                nv: 'nvd3'
-            }),
-            new webpack.DefinePlugin({NODE_ENV: JSON.stringify(NODE_ENV)})
-        ]
-    };
+export default {
+    cache: true,
+    node: {
+        fs: 'empty'
+    },
+
+    // Entry points.
+    entry: null,
+
+    // Output system.
+    output: null,
+
+    // Load plugins.
+    plugins: [
+        new webpack.ProvidePlugin({
+            $: 'jquery',
+            jQuery: 'jquery',
+            _: 'lodash',
+            nv: 'nvd3'
+        }),
+        new webpack.DefinePlugin({NODE_ENV: JSON.stringify(NODE_ENV)})
+    ]
 };

From 71c62f70df2b79f706b6cca157bccfbcea0ae5ef Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 21 Mar 2017 10:20:39 +0700
Subject: [PATCH 061/516] IGNITE-4830 Implemented better SQL errors handling.
 (cherry picked from commit fbb9940)

---
 .../app/modules/sql/sql.controller.js         | 84 ++++++++++++++-----
 .../frontend/views/sql/sql.tpl.pug            |  9 +-
 .../frontend/views/templates/message.tpl.pug  |  4 +-
 3 files changed, 71 insertions(+), 26 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 7ded2d5d99b6f..a3a5a5fb81dc7 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -18,7 +18,7 @@
 import paragraphRateTemplateUrl from 'views/sql/paragraph-rate.tpl.pug';
 import cacheMetadataTemplateUrl from 'views/sql/cache-metadata.tpl.pug';
 import chartSettingsTemplateUrl from 'views/sql/chart-settings.tpl.pug';
-import showQueryTemplateUrl from 'views/templates/message.tpl.pug';
+import messageTemplateUrl from 'views/templates/message.tpl.pug';
 
 // Time line X axis descriptor.
 const TIME_LINE = {value: -1, type: 'java.sql.Date', label: 'TIME_LINE'};
@@ -53,7 +53,7 @@ const _fullColName = (col) => {
 let paragraphId = 0;
 
 class Paragraph {
-    constructor($animate, $timeout, paragraph) {
+    constructor($animate, $timeout, JavaTypes, paragraph) {
         const self = this;
 
         self.id = 'paragraph-' + paragraphId++;
@@ -140,13 +140,36 @@ class Paragraph {
         }});
 
         Object.defineProperty(this, 'chartHistory', {value: []});
+
+        Object.defineProperty(this, 'error', {value: {
+            root: {},
+            message: ''
+        }});
+
+        this.setError = (err) => {
+            this.error.root = err;
+            this.error.message = err.message;
+
+            let cause = err;
+
+            while (_.nonNil(cause)) {
+                if (_.nonEmpty(cause.className) &&
+                    _.includes(['SQLException', 'JdbcSQLException'], JavaTypes.shortClassName(cause.className))) {
+                    this.error.message = cause.message;
+
+                    break;
+                }
+
+                cause = cause.cause;
+            }
+        };
     }
 
     resultType() {
         if (_.isNil(this.queryArgs))
             return null;
 
-        if (!_.isEmpty(this.errMsg))
+        if (_.nonEmpty(this.error.message))
             return 'error';
 
         if (_.isEmpty(this.rows))
@@ -172,7 +195,7 @@ class Paragraph {
     }
 
     queryExecuted() {
-        return !_.isEmpty(this.meta) || !_.isEmpty(this.errMsg);
+        return _.nonEmpty(this.meta) || _.nonEmpty(this.error.message);
     }
 
     scanExplain() {
@@ -184,17 +207,17 @@ class Paragraph {
     }
 
     chartColumnsConfigured() {
-        return !_.isEmpty(this.chartKeyCols) && !_.isEmpty(this.chartValCols);
+        return _.nonEmpty(this.chartKeyCols) && _.nonEmpty(this.chartValCols);
     }
 
     chartTimeLineEnabled() {
-        return !_.isEmpty(this.chartKeyCols) && _.eq(this.chartKeyCols[0], TIME_LINE);
+        return _.nonEmpty(this.chartKeyCols) && _.eq(this.chartKeyCols[0], TIME_LINE);
     }
 }
 
 // Controller for SQL notebook screen.
-export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData',
-    function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData) {
+export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'IgniteAgentMonitor', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData', 'JavaTypes',
+    function($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMonitor, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes) {
         const $ctrl = this;
 
         // Define template urls.
@@ -909,7 +932,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                     $scope.notebook.paragraphs = [];
 
                 $scope.notebook.paragraphs = _.map($scope.notebook.paragraphs,
-                    (paragraph) => new Paragraph($animate, $timeout, paragraph));
+                    (paragraph) => new Paragraph($animate, $timeout, JavaTypes, paragraph));
 
                 if (_.isEmpty($scope.notebook.paragraphs))
                     $scope.addQuery();
@@ -981,7 +1004,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
             ActivitiesData.post({ action: '/queries/add/query' });
 
-            const paragraph = new Paragraph($animate, $timeout, {
+            const paragraph = new Paragraph($animate, $timeout, JavaTypes, {
                 name: 'Query' + (sz === 0 ? '' : sz),
                 query: '',
                 pageSize: $scope.pageSizes[1],
@@ -1009,7 +1032,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
             ActivitiesData.post({ action: '/queries/add/scan' });
 
-            const paragraph = new Paragraph($animate, $timeout, {
+            const paragraph = new Paragraph($animate, $timeout, JavaTypes, {
                 name: 'Scan' + (sz === 0 ? '' : sz),
                 query: '',
                 pageSize: $scope.pageSizes[1],
@@ -1246,7 +1269,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
 
             paragraph.resNodeId = res.responseNodeId;
 
-            delete paragraph.errMsg;
+            paragraph.setError({message: ''});
 
             // Prepare explain results for display in table.
             if (paragraph.queryArgs.query && paragraph.queryArgs.query.startsWith('EXPLAIN') && res.rows) {
@@ -1341,7 +1364,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 .then((nid) => agentMonitor.query(nid, args.cacheName, args.query, args.nonCollocatedJoins,
                     args.enforceJoinOrder, !!args.localNid, args.pageSize))
                 .then(_processQueryResult.bind(this, paragraph, false))
-                .catch((err) => paragraph.errMsg = err.message);
+                .catch((err) => paragraph.setError(err));
         };
 
         const _tryStartRefresh = function(paragraph) {
@@ -1419,7 +1442,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                             _tryStartRefresh(paragraph);
                         })
                         .catch((err) => {
-                            paragraph.errMsg = err.message;
+                            paragraph.setError(err);
 
                             _showLoading(paragraph, false);
 
@@ -1468,7 +1491,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 })
                 .then(_processQueryResult.bind(this, paragraph, true))
                 .catch((err) => {
-                    paragraph.errMsg = err.message;
+                    paragraph.setError(err);
 
                     _showLoading(paragraph, false);
                 })
@@ -1506,7 +1529,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                         })
                         .then((res) => _processQueryResult(paragraph, true, res))
                         .catch((err) => {
-                            paragraph.errMsg = err.message;
+                            paragraph.setError(err);
 
                             _showLoading(paragraph, false);
                         });
@@ -1560,7 +1583,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                         delete paragraph.queryId;
                 })
                 .catch((err) => {
-                    paragraph.errMsg = err.message;
+                    paragraph.setError(err);
 
                     _showLoading(paragraph, false);
                 })
@@ -1756,15 +1779,36 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
                 }
                 else if (paragraph.queryArgs.query .startsWith('EXPLAIN ')) {
                     scope.title = 'Explain query';
-                    scope.content = [paragraph.queryArgs.query];
+                    scope.content = paragraph.queryArgs.query.split(/\r?\n/);
                 }
                 else {
                     scope.title = 'SQL query';
-                    scope.content = [paragraph.queryArgs.query];
+                    scope.content = paragraph.queryArgs.query.split(/\r?\n/);
+                }
+
+                // Show a basic modal from a controller
+                $modal({scope, templateUrl: messageTemplateUrl, placement: 'center', show: true});
+            }
+        };
+
+        $scope.showStackTrace = function(paragraph) {
+            if (!_.isNil(paragraph)) {
+                const scope = $scope.$new();
+
+                scope.title = 'Error details';
+                scope.content = [];
+
+                let cause = paragraph.error.root;
+
+                while (_.nonNil(cause)) {
+                    scope.content.push((scope.content.length > 0 ? '&nbsp;&nbsp;&nbsp;&nbsp;' : '') +
+                        '[' + JavaTypes.shortClassName(cause.className) + '] ' + cause.message);
+
+                    cause = cause.cause;
                 }
 
                 // Show a basic modal from a controller
-                $modal({scope, templateUrl: showQueryTemplateUrl, placement: 'center', show: true});
+                $modal({scope, templateUrl: messageTemplateUrl, placement: 'center', show: true});
             }
         };
     }
diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug
index bc134a67e5bce..701ee143af0ca 100644
--- a/modules/web-console/frontend/views/sql/sql.tpl.pug
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -199,7 +199,7 @@ mixin paragraph-scan
                 | Scan on selected node
 
         .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
-            .error(ng-switch-when='error') Error: {{paragraph.errMsg}}
+            .error(ng-switch-when='error') Error: {{paragraph.error.message}}
             .empty(ng-switch-when='empty') Result set is empty
             .table(ng-switch-when='table')
                 +table-result-heading-scan
@@ -247,7 +247,7 @@ mixin paragraph-query
             .pull-right
                 +query-settings
         .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
-            .error(ng-switch-when='error') Error: {{paragraph.errMsg}}
+            .error(ng-switch-when='error') Error: {{paragraph.error.message}}
             .empty(ng-switch-when='empty') Result set is empty
             .table(ng-switch-when='table')
                 +table-result-heading-query
@@ -255,9 +255,10 @@ mixin paragraph-query
             .chart(ng-switch-when='chart')
                 +chart-result
             .footer.clearfix
-                a.pull-left(ng-click='showResultQuery(paragraph)') Show query
+                a.pull-left(ng-show='paragraph.resultType() === "error"' ng-click='showStackTrace(paragraph)') Show error details
+                a.pull-left(ng-show='paragraph.resultType() !== "error"' ng-click='showResultQuery(paragraph)') Show query
 
-                -var nextVisibleCondition = 'paragraph.resultType() != "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
+                -var nextVisibleCondition = 'paragraph.resultType() !== "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
 
                 .pull-right(ng-show=`${nextVisibleCondition}` ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)')
                     i.fa.fa-chevron-circle-right
diff --git a/modules/web-console/frontend/views/templates/message.tpl.pug b/modules/web-console/frontend/views/templates/message.tpl.pug
index 6eff74bba5173..aa3615fbee0c6 100644
--- a/modules/web-console/frontend/views/templates/message.tpl.pug
+++ b/modules/web-console/frontend/views/templates/message.tpl.pug
@@ -22,7 +22,7 @@
                 h4.modal-title
                     i.fa.fa-info-circle
                     | {{title}}
-            .modal-body(ng-show='content')
-                p(ng-bind-html='content.join("<br/>")' style='text-align: left;')
+            .modal-body(ng-show='content' style='overflow: auto; max-height: 300px;')
+                p(ng-bind-html='content.join("<br/>")' style='text-align: left; white-space: nowrap;')
             .modal-footer
                 button.btn.btn-primary(id='confirm-btn-confirm' ng-click='$hide()') Ok

From 93faee3579389c5754e3b91bcd39cc88dcf39125 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 21 Mar 2017 11:07:15 +0700
Subject: [PATCH 062/516] Web Console: Update classnames.properties.

---
 .../resources/META-INF/classnames.properties    | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 7bebe24f8a51b..0d3e9ea16a078 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -24,6 +24,7 @@ org.apache.ignite.IgniteException
 org.apache.ignite.IgniteIllegalStateException
 org.apache.ignite.IgniteInterruptedException
 org.apache.ignite.IgniteState
+org.apache.ignite.IgniteSystemProperties$1
 org.apache.ignite.binary.BinaryInvalidTypeException
 org.apache.ignite.binary.BinaryObject
 org.apache.ignite.binary.BinaryObjectException
@@ -382,9 +383,9 @@ org.apache.ignite.internal.processors.cache.GridCacheAdapter$12
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$13
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$14
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$15
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$16
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$16$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$17
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$18$1
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$18
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$2
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$26$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$28
@@ -842,13 +843,13 @@ org.apache.ignite.internal.processors.cache.query.GridCacheQueryAdapter$ScanQuer
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryDetailMetricsAdapter
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryFutureAdapter$1
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryFutureAdapter$2
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$10
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$11
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$12
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$13
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$14
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$15
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$16
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$17
-org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$18$1
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$17$1
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$4$1
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$4$2
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$5
@@ -1738,6 +1739,7 @@ org.apache.ignite.internal.visor.compute.VisorComputeToggleMonitoringTask
 org.apache.ignite.internal.visor.compute.VisorComputeToggleMonitoringTask$VisorComputeToggleMonitoringJob
 org.apache.ignite.internal.visor.compute.VisorGatewayTask
 org.apache.ignite.internal.visor.compute.VisorGatewayTask$VisorGatewayJob
+org.apache.ignite.internal.visor.compute.VisorGatewayTask$VisorGatewayJob$1
 org.apache.ignite.internal.visor.debug.VisorThreadDumpTask
 org.apache.ignite.internal.visor.debug.VisorThreadDumpTask$VisorDumpThreadJob
 org.apache.ignite.internal.visor.debug.VisorThreadInfo
@@ -1817,6 +1819,10 @@ org.apache.ignite.internal.visor.node.VisorRestConfiguration
 org.apache.ignite.internal.visor.node.VisorSegmentationConfiguration
 org.apache.ignite.internal.visor.node.VisorSpisConfiguration
 org.apache.ignite.internal.visor.node.VisorTransactionConfiguration
+org.apache.ignite.internal.visor.query.VisorCancelQueriesTask
+org.apache.ignite.internal.visor.query.VisorCancelQueriesTask$VisorCancelQueriesJob
+org.apache.ignite.internal.visor.query.VisorCollectRunningQueriesTask
+org.apache.ignite.internal.visor.query.VisorCollectRunningQueriesTask$VisorCollectRunningQueriesJob
 org.apache.ignite.internal.visor.query.VisorQueryArg
 org.apache.ignite.internal.visor.query.VisorQueryArgV2
 org.apache.ignite.internal.visor.query.VisorQueryArgV3
@@ -1830,6 +1836,7 @@ org.apache.ignite.internal.visor.query.VisorQueryResult
 org.apache.ignite.internal.visor.query.VisorQueryResultEx
 org.apache.ignite.internal.visor.query.VisorQueryScanSubstringFilter
 org.apache.ignite.internal.visor.query.VisorQueryTask
+org.apache.ignite.internal.visor.query.VisorRunningQuery
 org.apache.ignite.internal.visor.service.VisorCancelServiceTask
 org.apache.ignite.internal.visor.service.VisorCancelServiceTask$VisorCancelServiceJob
 org.apache.ignite.internal.visor.service.VisorServiceDescriptor

From 45812566694d016c07b47fa589bbba612c26f824 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 21 Mar 2017 11:23:18 +0700
Subject: [PATCH 063/516] Web Console: minor fix. (cherry picked from commit
 92bce6e)

---
 modules/web-console/frontend/app/modules/sql/sql.controller.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index a3a5a5fb81dc7..075bd559ee1e2 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -154,7 +154,7 @@ class Paragraph {
 
             while (_.nonNil(cause)) {
                 if (_.nonEmpty(cause.className) &&
-                    _.includes(['SQLException', 'JdbcSQLException'], JavaTypes.shortClassName(cause.className))) {
+                    _.includes(['SQLException', 'JdbcSQLException', 'QueryCancelledException'], JavaTypes.shortClassName(cause.className))) {
                     this.error.message = cause.message;
 
                     break;

From 84bd00b70caf2ae0b1de97e25536f6fa2e9a8c47 Mon Sep 17 00:00:00 2001
From: oleg-ostanin <oostanin@gridgain.com>
Date: Tue, 21 Mar 2017 15:12:35 +0300
Subject: [PATCH 064/516] IGNITE-4822 Fixed change jvm options for benchmarks

---
 .../config/benchmark-atomic.properties        | 18 +++++++--------
 .../config/benchmark-bin-identity.properties  | 10 +++-----
 .../config/benchmark-cache-load.properties    | 10 +++-----
 .../config/benchmark-client-mode.properties   | 10 +++-----
 .../config/benchmark-compute.properties       | 18 +++++++--------
 .../config/benchmark-failover.properties      | 18 ++++++---------
 .../config/benchmark-full.properties          | 18 ++++++---------
 .../config/benchmark-multicast.properties     | 10 +++-----
 .../benchmark-put-indexed-val.properties      | 18 +++++++--------
 .../benchmark-query-put-separated.properties  | 10 +++-----
 .../config/benchmark-query.properties         | 18 +++++++--------
 .../config/benchmark-remote-sample.properties | 18 +++++++--------
 .../config/benchmark-remote.properties        | 18 ++++++---------
 .../config/benchmark-sample.properties        | 18 +++++++--------
 .../config/benchmark-sql-dml.properties       | 18 +++++++--------
 .../config/benchmark-store.properties         | 10 +++-----
 .../yardstick/config/benchmark-tx.properties  | 18 +++++++--------
 modules/yardstick/config/benchmark.properties | 18 ++++++---------
 .../test-max-int-values-offheap.properties    | 23 +++++++------------
 .../test-max-int-values-onheap.properties     | 23 +++++++------------
 .../test-max-int-values-swap.properties       | 23 +++++++------------
 21 files changed, 134 insertions(+), 211 deletions(-)

diff --git a/modules/yardstick/config/benchmark-atomic.properties b/modules/yardstick/config/benchmark-atomic.properties
index 722638490d866..967a603e7d02a 100644
--- a/modules/yardstick/config/benchmark-atomic.properties
+++ b/modules/yardstick/config/benchmark-atomic.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-bin-identity.properties b/modules/yardstick/config/benchmark-bin-identity.properties
index 6468d8f9f7142..d5d18b6d02276 100644
--- a/modules/yardstick/config/benchmark-bin-identity.properties
+++ b/modules/yardstick/config/benchmark-bin-identity.properties
@@ -24,17 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-cache-load.properties b/modules/yardstick/config/benchmark-cache-load.properties
index 12e07c33a0705..9c1ab093d6788 100644
--- a/modules/yardstick/config/benchmark-cache-load.properties
+++ b/modules/yardstick/config/benchmark-cache-load.properties
@@ -21,17 +21,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-client-mode.properties b/modules/yardstick/config/benchmark-client-mode.properties
index 0723fcb9cc03f..ad501f118dd79 100644
--- a/modules/yardstick/config/benchmark-client-mode.properties
+++ b/modules/yardstick/config/benchmark-client-mode.properties
@@ -24,17 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 #Ignite version
 ver="RELEASE-"
diff --git a/modules/yardstick/config/benchmark-compute.properties b/modules/yardstick/config/benchmark-compute.properties
index 66cb93030a39a..df35c54e1de88 100644
--- a/modules/yardstick/config/benchmark-compute.properties
+++ b/modules/yardstick/config/benchmark-compute.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-failover.properties b/modules/yardstick/config/benchmark-failover.properties
index 4c6a12a61dec4..a85d165d3f2bb 100644
--- a/modules/yardstick/config/benchmark-failover.properties
+++ b/modules/yardstick/config/benchmark-failover.properties
@@ -27,17 +27,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false -ea"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
-  -XX:+PrintGCDetails \
-  -verbose:gc \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-full.properties b/modules/yardstick/config/benchmark-full.properties
index b64612ec56435..96da0a6e4d390 100644
--- a/modules/yardstick/config/benchmark-full.properties
+++ b/modules/yardstick/config/benchmark-full.properties
@@ -24,17 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
-  -XX:+PrintGCDetails \
-  -verbose:gc \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-multicast.properties b/modules/yardstick/config/benchmark-multicast.properties
index c10d0c67da605..7f76495291d6c 100644
--- a/modules/yardstick/config/benchmark-multicast.properties
+++ b/modules/yardstick/config/benchmark-multicast.properties
@@ -24,17 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-put-indexed-val.properties b/modules/yardstick/config/benchmark-put-indexed-val.properties
index 0d699c5492d8a..152d72f8f89a9 100644
--- a/modules/yardstick/config/benchmark-put-indexed-val.properties
+++ b/modules/yardstick/config/benchmark-put-indexed-val.properties
@@ -21,16 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/benchmark-query-put-separated.properties b/modules/yardstick/config/benchmark-query-put-separated.properties
index 500e4faae4850..b4437bf368bc8 100644
--- a/modules/yardstick/config/benchmark-query-put-separated.properties
+++ b/modules/yardstick/config/benchmark-query-put-separated.properties
@@ -24,17 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-query.properties b/modules/yardstick/config/benchmark-query.properties
index af9b07e22725c..9192512b5e419 100644
--- a/modules/yardstick/config/benchmark-query.properties
+++ b/modules/yardstick/config/benchmark-query.properties
@@ -21,16 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/benchmark-remote-sample.properties b/modules/yardstick/config/benchmark-remote-sample.properties
index 5f158ac9a6cb0..7e97f43f6531b 100644
--- a/modules/yardstick/config/benchmark-remote-sample.properties
+++ b/modules/yardstick/config/benchmark-remote-sample.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-remote.properties b/modules/yardstick/config/benchmark-remote.properties
index 0cda79e7ef44e..4d671d3ad92ff 100644
--- a/modules/yardstick/config/benchmark-remote.properties
+++ b/modules/yardstick/config/benchmark-remote.properties
@@ -24,17 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
-  -XX:+PrintGCDetails \
-  -verbose:gc \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-sample.properties b/modules/yardstick/config/benchmark-sample.properties
index 1932ae7b7ba71..81e9a1b52ad25 100644
--- a/modules/yardstick/config/benchmark-sample.properties
+++ b/modules/yardstick/config/benchmark-sample.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-sql-dml.properties b/modules/yardstick/config/benchmark-sql-dml.properties
index a8bb4537765c7..faa62c9dd5bf9 100644
--- a/modules/yardstick/config/benchmark-sql-dml.properties
+++ b/modules/yardstick/config/benchmark-sql-dml.properties
@@ -21,16 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/benchmark-store.properties b/modules/yardstick/config/benchmark-store.properties
index ea8487f749096..2f89d0e3257c6 100644
--- a/modules/yardstick/config/benchmark-store.properties
+++ b/modules/yardstick/config/benchmark-store.properties
@@ -26,17 +26,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-tx.properties b/modules/yardstick/config/benchmark-tx.properties
index 4744dee0311ac..cc48c5309599d 100644
--- a/modules/yardstick/config/benchmark-tx.properties
+++ b/modules/yardstick/config/benchmark-tx.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes, comma separated.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark.properties b/modules/yardstick/config/benchmark.properties
index 1b5d5fe966020..cccbf2afb482b 100644
--- a/modules/yardstick/config/benchmark.properties
+++ b/modules/yardstick/config/benchmark.properties
@@ -24,17 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
-  -XX:+PrintGCDetails \
-  -verbose:gc \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/test-max-int-values-offheap.properties b/modules/yardstick/config/test-max-int-values-offheap.properties
index 42769ec1a013b..838986da1bbda 100644
--- a/modules/yardstick/config/test-max-int-values-offheap.properties
+++ b/modules/yardstick/config/test-max-int-values-offheap.properties
@@ -21,21 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+PrintGCDetails \
-  -XX:-PrintGCTimeStamps \
-  -verbose:gc \
-  -Xmx8g \
-  -Xms8g \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=1g \
-  -XX:MaxNewSize=1g \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/test-max-int-values-onheap.properties b/modules/yardstick/config/test-max-int-values-onheap.properties
index 0faf5f3d58c08..c83f46997a2ca 100644
--- a/modules/yardstick/config/test-max-int-values-onheap.properties
+++ b/modules/yardstick/config/test-max-int-values-onheap.properties
@@ -21,21 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+PrintGCDetails \
-  -XX:-PrintGCTimeStamps \
-  -verbose:gc \
-  -Xmx92g \
-  -Xms32g \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=4g \
-  -XX:MaxNewSize=4g \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms92g \
+-Xmx32g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/test-max-int-values-swap.properties b/modules/yardstick/config/test-max-int-values-swap.properties
index 900e8ac1b77cb..c462ada25cd46 100644
--- a/modules/yardstick/config/test-max-int-values-swap.properties
+++ b/modules/yardstick/config/test-max-int-values-swap.properties
@@ -21,21 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+PrintGCDetails \
-  -XX:-PrintGCTimeStamps \
-  -verbose:gc \
-  -Xmx8g \
-  -Xms8g \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=1g \
-  -XX:MaxNewSize=1g \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.

From f4232e9557fca89065f4ee424afab940265e2d1b Mon Sep 17 00:00:00 2001
From: oleg-ostanin <oostanin@gridgain.com>
Date: Tue, 21 Mar 2017 15:32:33 +0300
Subject: [PATCH 065/516] IGNITE-4822 Fixed change jvm options for benchmarks

---
 .../config/benchmark-atomic.properties        | 18 +++++++-------
 .../config/benchmark-bin-identity.properties  |  9 ++-----
 .../config/benchmark-cache-load.properties    |  9 ++-----
 .../config/benchmark-client-mode.properties   |  9 ++-----
 .../config/benchmark-compute.properties       | 18 +++++++-------
 .../config/benchmark-failover.properties      | 18 ++++++--------
 .../config/benchmark-full.properties          | 18 ++++++--------
 .../config/benchmark-multicast.properties     |  9 ++-----
 .../benchmark-put-indexed-val.properties      | 18 +++++++-------
 .../benchmark-query-put-separated.properties  |  9 ++-----
 .../config/benchmark-query.properties         | 18 +++++++-------
 .../config/benchmark-sql-dml.properties       | 18 +++++++-------
 .../config/benchmark-store.properties         |  9 ++-----
 .../yardstick/config/benchmark-tx.properties  | 18 +++++++-------
 modules/yardstick/config/benchmark.properties | 18 ++++++--------
 .../test-max-int-values-offheap.properties    | 24 +++++++------------
 .../test-max-int-values-onheap.properties     | 24 +++++++------------
 .../test-max-int-values-swap.properties       | 24 +++++++------------
 18 files changed, 105 insertions(+), 183 deletions(-)

diff --git a/modules/yardstick/config/benchmark-atomic.properties b/modules/yardstick/config/benchmark-atomic.properties
index fc827e0edb1e6..674f5a15b3537 100644
--- a/modules/yardstick/config/benchmark-atomic.properties
+++ b/modules/yardstick/config/benchmark-atomic.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-bin-identity.properties b/modules/yardstick/config/benchmark-bin-identity.properties
index 3d754c55a4b06..13cbc61077b53 100644
--- a/modules/yardstick/config/benchmark-bin-identity.properties
+++ b/modules/yardstick/config/benchmark-bin-identity.properties
@@ -24,18 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-cache-load.properties b/modules/yardstick/config/benchmark-cache-load.properties
index 7312cb6acefd5..95e9315b3e9db 100644
--- a/modules/yardstick/config/benchmark-cache-load.properties
+++ b/modules/yardstick/config/benchmark-cache-load.properties
@@ -21,18 +21,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-client-mode.properties b/modules/yardstick/config/benchmark-client-mode.properties
index f7c83472cf570..0c3dedb7a481c 100644
--- a/modules/yardstick/config/benchmark-client-mode.properties
+++ b/modules/yardstick/config/benchmark-client-mode.properties
@@ -24,18 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 #Ignite version
 ver="RELEASE-"
diff --git a/modules/yardstick/config/benchmark-compute.properties b/modules/yardstick/config/benchmark-compute.properties
index a4816bdd008c6..30a4b09bb0686 100644
--- a/modules/yardstick/config/benchmark-compute.properties
+++ b/modules/yardstick/config/benchmark-compute.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark-failover.properties b/modules/yardstick/config/benchmark-failover.properties
index 352e2d4058569..d4e17ee40780e 100644
--- a/modules/yardstick/config/benchmark-failover.properties
+++ b/modules/yardstick/config/benchmark-failover.properties
@@ -28,17 +28,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false -ea"
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
   -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -verbose:gc \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-full.properties b/modules/yardstick/config/benchmark-full.properties
index da3ae9ff8a716..9d9ea90c8acd8 100644
--- a/modules/yardstick/config/benchmark-full.properties
+++ b/modules/yardstick/config/benchmark-full.properties
@@ -25,17 +25,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
   -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -verbose:gc \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-multicast.properties b/modules/yardstick/config/benchmark-multicast.properties
index 3b31745b1d79d..e979070abf74c 100644
--- a/modules/yardstick/config/benchmark-multicast.properties
+++ b/modules/yardstick/config/benchmark-multicast.properties
@@ -24,18 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-put-indexed-val.properties b/modules/yardstick/config/benchmark-put-indexed-val.properties
index e81ae6f89e235..49be7b9cf941d 100644
--- a/modules/yardstick/config/benchmark-put-indexed-val.properties
+++ b/modules/yardstick/config/benchmark-put-indexed-val.properties
@@ -21,16 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/benchmark-query-put-separated.properties b/modules/yardstick/config/benchmark-query-put-separated.properties
index 6d2ce33e03d26..0e73bbcc927c1 100644
--- a/modules/yardstick/config/benchmark-query-put-separated.properties
+++ b/modules/yardstick/config/benchmark-query-put-separated.properties
@@ -24,18 +24,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-query.properties b/modules/yardstick/config/benchmark-query.properties
index 486d00e14327f..44184f2997a30 100644
--- a/modules/yardstick/config/benchmark-query.properties
+++ b/modules/yardstick/config/benchmark-query.properties
@@ -21,16 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/benchmark-sql-dml.properties b/modules/yardstick/config/benchmark-sql-dml.properties
index 2ce2e1fc7a261..7aad883cb3e52 100644
--- a/modules/yardstick/config/benchmark-sql-dml.properties
+++ b/modules/yardstick/config/benchmark-sql-dml.properties
@@ -21,16 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/benchmark-store.properties b/modules/yardstick/config/benchmark-store.properties
index 746fa11ffcc42..4ebf8a60a1691 100644
--- a/modules/yardstick/config/benchmark-store.properties
+++ b/modules/yardstick/config/benchmark-store.properties
@@ -26,18 +26,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
+-Xms6g \
+-Xmx6g \
 -Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
 -XX:+UseConcMarkSweepGC \
--XX:+UseTLAB \
--XX:NewSize=128m \
--XX:MaxNewSize=128m \
--XX:MaxTenuringThreshold=0 \
--XX:SurvivorRatio=1024 \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=60 \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/benchmark-tx.properties b/modules/yardstick/config/benchmark-tx.properties
index 0d5bb02dd3e1a..8a3620697f824 100644
--- a/modules/yardstick/config/benchmark-tx.properties
+++ b/modules/yardstick/config/benchmark-tx.properties
@@ -22,16 +22,14 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 # JVM_OPTS=${JVM_OPTS}" \
-#  -XX:+UseParNewGC \
-#  -XX:+UseConcMarkSweepGC \
-#  -XX:+UseTLAB \
-#  -XX:NewSize=128m \
-#  -XX:MaxNewSize=128m \
-#  -XX:MaxTenuringThreshold=0 \
-#  -XX:SurvivorRatio=1024 \
-#  -XX:+UseCMSInitiatingOccupancyOnly \
-#  -XX:CMSInitiatingOccupancyFraction=60 \
-#"
+# -Xms6g \
+# -Xmx6g \
+# -Xloggc:./gc${now0}.log \
+# -XX:+PrintGCDetails \
+# -verbose:gc \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# "
 
 # List of default probes, comma separated.
 # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
diff --git a/modules/yardstick/config/benchmark.properties b/modules/yardstick/config/benchmark.properties
index cfc14999751d9..de0cb5aceac1b 100644
--- a/modules/yardstick/config/benchmark.properties
+++ b/modules/yardstick/config/benchmark.properties
@@ -25,17 +25,13 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
   -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -verbose:gc \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=128m \
-  -XX:MaxNewSize=128m \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+-Xms6g \
+-Xmx6g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 #Ignite version
diff --git a/modules/yardstick/config/test-max-int-values-offheap.properties b/modules/yardstick/config/test-max-int-values-offheap.properties
index 5b4da209db71c..fb501d7ae0c75 100644
--- a/modules/yardstick/config/test-max-int-values-offheap.properties
+++ b/modules/yardstick/config/test-max-int-values-offheap.properties
@@ -21,22 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -XX:-PrintGCTimeStamps \
-  -verbose:gc \
-  -Xmx8g \
-  -Xms8g \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=1g \
-  -XX:MaxNewSize=1g \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+    -Xms8g \
+    -Xmx8g \
+    -Xloggc:./gc${now0}.log \
+    -XX:+PrintGCDetails \
+    -verbose:gc \
+    -XX:+UseParNewGC \
+    -XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/test-max-int-values-onheap.properties b/modules/yardstick/config/test-max-int-values-onheap.properties
index d29800aa2ce70..6616003be573e 100644
--- a/modules/yardstick/config/test-max-int-values-onheap.properties
+++ b/modules/yardstick/config/test-max-int-values-onheap.properties
@@ -21,22 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -XX:-PrintGCTimeStamps \
-  -verbose:gc \
-  -Xmx92g \
-  -Xms32g \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=4g \
-  -XX:MaxNewSize=4g \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms92g \
+-Xmx32g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.
diff --git a/modules/yardstick/config/test-max-int-values-swap.properties b/modules/yardstick/config/test-max-int-values-swap.properties
index 203f0040a78a3..cecad8b59b5bd 100644
--- a/modules/yardstick/config/test-max-int-values-swap.properties
+++ b/modules/yardstick/config/test-max-int-values-swap.properties
@@ -21,22 +21,14 @@
 # JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
-JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
-  -Xloggc:./gc${now0}.log \
-  -XX:+PrintGCDetails \
-  -XX:-PrintGCTimeStamps \
-  -verbose:gc \
-  -Xmx8g \
-  -Xms8g \
-  -XX:+UseParNewGC \
-  -XX:+UseConcMarkSweepGC \
-  -XX:+UseTLAB \
-  -XX:NewSize=1g \
-  -XX:MaxNewSize=1g \
-  -XX:MaxTenuringThreshold=0 \
-  -XX:SurvivorRatio=1024 \
-  -XX:+UseCMSInitiatingOccupancyOnly \
-  -XX:CMSInitiatingOccupancyFraction=60 \
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
 "
 
 # List of default probes.

From ab3b2926213291a45305c21e7efe066855c1342f Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 21 Mar 2017 17:47:09 +0300
Subject: [PATCH 066/516] IGNITE-4200: Added copying of the C++ binaries.

---
 assembly/release-fabric-base.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml
index 7b3d8cf21e5f5..7f05c305ed4b2 100644
--- a/assembly/release-fabric-base.xml
+++ b/assembly/release-fabric-base.xml
@@ -174,6 +174,12 @@
             <outputDirectory>/platforms/cpp/docs</outputDirectory>
         </fileSet>
 
+        <!-- Move CPP binaries. -->
+        <fileSet>
+            <directory>modules/platforms/cpp/bin</directory>
+            <outputDirectory>/platforms/cpp/bin</outputDirectory>
+        </fileSet>
+
         <!-- Other files. -->
         <fileSet>
             <directory>bin</directory>

From e0c012d977b6db13dfdf2fb8347677998287c1e4 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 21 Mar 2017 17:50:06 +0300
Subject: [PATCH 067/516] IGNITE-4200: Added copying of the C++ binaries.

---
 assembly/release-fabric-base.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml
index 7b3d8cf21e5f5..7f05c305ed4b2 100644
--- a/assembly/release-fabric-base.xml
+++ b/assembly/release-fabric-base.xml
@@ -174,6 +174,12 @@
             <outputDirectory>/platforms/cpp/docs</outputDirectory>
         </fileSet>
 
+        <!-- Move CPP binaries. -->
+        <fileSet>
+            <directory>modules/platforms/cpp/bin</directory>
+            <outputDirectory>/platforms/cpp/bin</outputDirectory>
+        </fileSet>
+
         <!-- Other files. -->
         <fileSet>
             <directory>bin</directory>

From 6e2bc4bfb0b200e59c9fc3cf4fcbd408d52acb9c Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 21 Mar 2017 17:51:40 +0300
Subject: [PATCH 068/516] IGNITE-4200: Added copying of the C++ binaries.

---
 assembly/release-fabric-base.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml
index c008083221a30..78c77770e885a 100644
--- a/assembly/release-fabric-base.xml
+++ b/assembly/release-fabric-base.xml
@@ -174,6 +174,12 @@
             <outputDirectory>/platforms/cpp/docs</outputDirectory>
         </fileSet>
 
+        <!-- Move CPP binaries. -->
+        <fileSet>
+            <directory>modules/platforms/cpp/bin</directory>
+            <outputDirectory>/platforms/cpp/bin</outputDirectory>
+        </fileSet>
+
         <!-- Other files. -->
         <fileSet>
             <directory>bin</directory>

From 8b3860f5bd9518985a8a516cd14ff24a3a615f4c Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 21 Mar 2017 17:54:51 +0300
Subject: [PATCH 069/516] IGNITE-4200: Added copying of the C++ binaries.

---
 assembly/release-fabric-base.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml
index d7aa0a2ec2294..97222c94c635a 100644
--- a/assembly/release-fabric-base.xml
+++ b/assembly/release-fabric-base.xml
@@ -179,6 +179,12 @@
             <outputDirectory>/platforms/cpp/docs</outputDirectory>
         </fileSet>
 
+        <!-- Move CPP binaries. -->
+        <fileSet>
+            <directory>modules/platforms/cpp/bin</directory>
+            <outputDirectory>/platforms/cpp/bin</outputDirectory>
+        </fileSet>
+
         <!-- Other files. -->
         <fileSet>
             <directory>bin</directory>

From 8273e6703944ea50b229a512398ac741eb713073 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Tue, 21 Mar 2017 18:04:01 +0300
Subject: [PATCH 070/516] IGNITE-4779 Missed discovery data snapshot during
 exchange processing (Backport from 2.0)

---
 .../managers/discovery/DiscoCache.java        | 310 ++++++++
 .../discovery/GridDiscoveryManager.java       | 710 +++++-------------
 .../eventstorage/DiscoveryEventListener.java  |  33 +
 .../eventstorage/GridEventStorageManager.java | 162 +++-
 .../affinity/GridAffinityAssignmentCache.java |   7 +-
 .../cache/CacheAffinitySharedManager.java     |  35 +-
 .../cache/GridCacheAffinityManager.java       |   2 +-
 .../GridCachePartitionExchangeManager.java    |  64 +-
 .../dht/GridClientPartitionTopology.java      |  20 +-
 .../dht/GridDhtAssignmentFetchFuture.java     |   7 +-
 .../dht/GridDhtPartitionTopologyImpl.java     |  44 +-
 .../dht/atomic/GridDhtAtomicCache.java        |   4 +-
 .../GridDhtPartitionsExchangeFuture.java      |  33 +-
 .../service/GridServiceProcessor.java         |  21 +-
 ...ridDiscoveryManagerAliveCacheSelfTest.java |  64 +-
 ...ridDiscoveryManagerAttributesSelfTest.java |  14 +-
 .../GridDiscoveryManagerSelfTest.java         | 214 ------
 .../IgniteTopologyPrintFormatSelfTest.java    |   8 +-
 .../testsuites/IgniteKernalSelfTestSuite.java |   3 -
 19 files changed, 854 insertions(+), 901 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/DiscoveryEventListener.java
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
new file mode 100644
index 0000000000000..5247ac1b28091
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.P1;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class DiscoCache {
+    /** Local node. */
+    private final ClusterNode loc;
+
+    /** Remote nodes. */
+    private final List<ClusterNode> rmtNodes;
+
+    /** All nodes. */
+    private final List<ClusterNode> allNodes;
+
+    /** All server nodes. */
+    private final List<ClusterNode> srvNodes;
+
+    /** Daemon nodes. */
+    private final List<ClusterNode> daemonNodes;
+
+    /** All server nodes. */
+    private final List<ClusterNode> srvNodesWithCaches;
+
+    /** All nodes with at least one cache configured. */
+    @GridToStringInclude
+    private final List<ClusterNode> allNodesWithCaches;
+
+    /** All remote nodes with at least one cache configured. */
+    @GridToStringInclude
+    private final List<ClusterNode> rmtNodesWithCaches;
+
+    /** Cache nodes by cache name. */
+    @GridToStringInclude
+    private final Map<Integer, List<ClusterNode>> allCacheNodes;
+
+    /** Affinity cache nodes by cache name. */
+    @GridToStringInclude
+    private final Map<Integer, List<ClusterNode>> affCacheNodes;
+
+    /** Node map. */
+    private final Map<UUID, ClusterNode> nodeMap;
+
+    /** Caches where at least one node has near cache enabled. */
+    @GridToStringInclude
+    private final Set<Integer> nearEnabledCaches;
+
+    /** Alive nodes. */
+    private final Set<UUID> alives = new GridConcurrentHashSet<>();
+
+    /**
+     * @param loc Local node.
+     * @param rmtNodes Remote nodes.
+     * @param allNodes All nodes.
+     * @param srvNodes Server nodes.
+     * @param daemonNodes Daemon nodes.
+     * @param srvNodesWithCaches Server nodes with at least one cache configured.
+     * @param allNodesWithCaches All nodes with at least one cache configured.
+     * @param rmtNodesWithCaches Remote nodes with at least one cache configured.
+     * @param allCacheNodes Cache nodes by cache name.
+     * @param affCacheNodes Affinity cache nodes by cache name.
+     * @param nodeMap Node map.
+     * @param nearEnabledCaches Caches where at least one node has near cache enabled.
+     * @param alives Alive nodes.
+     */
+    DiscoCache(ClusterNode loc,
+        List<ClusterNode> rmtNodes,
+        List<ClusterNode> allNodes,
+        List<ClusterNode> srvNodes,
+        List<ClusterNode> daemonNodes,
+        List<ClusterNode> srvNodesWithCaches,
+        List<ClusterNode> allNodesWithCaches,
+        List<ClusterNode> rmtNodesWithCaches,
+        Map<Integer, List<ClusterNode>> allCacheNodes,
+        Map<Integer, List<ClusterNode>> affCacheNodes,
+        Map<UUID, ClusterNode> nodeMap,
+        Set<Integer> nearEnabledCaches,
+        Set<UUID> alives) {
+        this.loc = loc;
+        this.rmtNodes = rmtNodes;
+        this.allNodes = allNodes;
+        this.srvNodes = srvNodes;
+        this.daemonNodes = daemonNodes;
+        this.srvNodesWithCaches = srvNodesWithCaches;
+        this.allNodesWithCaches = allNodesWithCaches;
+        this.rmtNodesWithCaches = rmtNodesWithCaches;
+        this.allCacheNodes = allCacheNodes;
+        this.affCacheNodes = affCacheNodes;
+        this.nodeMap = nodeMap;
+        this.nearEnabledCaches = nearEnabledCaches;
+        this.alives.addAll(alives);
+    }
+
+    /** @return Local node. */
+    public ClusterNode localNode() {
+        return loc;
+    }
+
+    /** @return Remote nodes. */
+    public List<ClusterNode> remoteNodes() {
+        return rmtNodes;
+    }
+
+    /** @return All nodes. */
+    public List<ClusterNode> allNodes() {
+        return allNodes;
+    }
+
+    /** @return Server nodes. */
+    public List<ClusterNode> serverNodes() {
+        return srvNodes;
+    }
+
+    /** @return Daemon nodes. */
+    public List<ClusterNode> daemonNodes() {
+        return daemonNodes;
+    }
+
+    /** @return Server nodes with at least one cache configured. */
+    public List<ClusterNode> serverNodesWithCaches() {
+        return srvNodesWithCaches;
+    }
+
+    /**
+     * Gets all remote nodes that have at least one cache configured.
+     *
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> remoteNodesWithCaches() {
+        return rmtNodesWithCaches;
+    }
+
+    /**
+     * Gets collection of nodes with at least one cache configured.
+     *
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> allNodesWithCaches() {
+        return allNodesWithCaches;
+    }
+
+    /**
+     * Gets collection of server nodes with at least one cache configured.
+     *
+     * @return Collection of nodes.
+     */
+    public Collection<ClusterNode> aliveServerNodes() {
+        return F.view(serverNodes(), new P1<ClusterNode>() {
+            @Override public boolean apply(ClusterNode node) {
+                return alives.contains(node.id());
+            }
+        });
+    }
+
+    /**
+     * Gets collection of server nodes with at least one cache configured.
+     *
+     * @return Collection of nodes.
+     */
+    public Collection<ClusterNode> aliveServerNodesWithCaches() {
+        return F.view(serverNodesWithCaches(), new P1<ClusterNode>() {
+            @Override public boolean apply(ClusterNode node) {
+                return alives.contains(node.id());
+            }
+        });
+    }
+
+    /**
+     * @return Oldest alive server node.
+     */
+    public @Nullable ClusterNode oldestAliveServerNode(){
+        Iterator<ClusterNode> it = aliveServerNodes().iterator();
+        return it.hasNext() ? it.next() : null;
+    }
+
+    /**
+     * @return Oldest alive server node with at least one cache configured.
+     */
+    public @Nullable ClusterNode oldestAliveServerNodeWithCache(){
+        Iterator<ClusterNode> it = aliveServerNodesWithCaches().iterator();
+        return it.hasNext() ? it.next() : null;
+    }
+
+    /**
+     * Gets all nodes that have cache with given name.
+     *
+     * @param cacheName Cache name.
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> cacheNodes(@Nullable String cacheName) {
+        return cacheNodes(CU.cacheId(cacheName));
+    }
+
+    /**
+     * Gets all nodes that have cache with given ID.
+     *
+     * @param cacheId Cache ID.
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> cacheNodes(Integer cacheId) {
+        return emptyIfNull(allCacheNodes.get(cacheId));
+    }
+
+    /**
+     * Gets all nodes that have cache with given ID and should participate in affinity calculation. With
+     * partitioned cache nodes with near-only cache do not participate in affinity node calculation.
+     *
+     * @param cacheName Cache name.
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> cacheAffinityNodes(@Nullable String cacheName) {
+        return cacheAffinityNodes(CU.cacheId(cacheName));
+    }
+
+    /**
+     * Gets all nodes that have cache with given ID and should participate in affinity calculation. With
+     * partitioned cache nodes with near-only cache do not participate in affinity node calculation.
+     *
+     * @param cacheId Cache ID.
+     * @return Collection of nodes.
+     */
+    public List<ClusterNode> cacheAffinityNodes(int cacheId) {
+        return emptyIfNull(affCacheNodes.get(cacheId));
+    }
+
+    /**
+     * Checks if cache with given ID has at least one node with near cache enabled.
+     *
+     * @param cacheId Cache ID.
+     * @return {@code True} if cache with given name has at least one node with near cache enabled.
+     */
+    public boolean hasNearCache(int cacheId) {
+        return nearEnabledCaches.contains(cacheId);
+    }
+
+    /**
+     * @param id Node ID.
+     * @return Node.
+     */
+    public @Nullable ClusterNode node(UUID id) {
+        return nodeMap.get(id);
+    }
+
+    /**
+     * Removes left node from alives lists.
+     *
+     * @param rmvd Removed node.
+     */
+    public void updateAlives(ClusterNode rmvd) {
+        alives.remove(rmvd.id());
+    }
+
+    /**
+     * Removes left nodes from cached alives lists.
+     *
+     * @param discovery Discovery manager.
+     */
+    public void updateAlives(GridDiscoveryManager discovery) {
+        for (UUID alive : alives) {
+            if (!discovery.alive(alive))
+                alives.remove(alive);
+        }
+    }
+
+    /**
+     * @param nodes Cluster nodes.
+     * @return Empty collection if nodes list is {@code null}
+     */
+    private List<ClusterNode> emptyIfNull(List<ClusterNode> nodes) {
+        return nodes == null ? Collections.<ClusterNode>emptyList() : nodes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DiscoCache.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 9aa4db1e0042f..80549dc08228b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -35,14 +35,12 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.NavigableMap;
 import java.util.Set;
-import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -74,24 +72,23 @@
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.jobmetrics.GridJobMetrics;
 import org.apache.ignite.internal.processors.security.SecurityContext;
 import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
-import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.lang.GridTuple5;
+import org.apache.ignite.internal.util.lang.GridTuple6;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.P1;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -99,7 +96,6 @@
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.segmentation.SegmentationPolicy;
@@ -113,6 +109,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
 import org.apache.ignite.thread.IgniteThread;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
@@ -233,7 +230,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
     private long segChkFreq;
 
     /** Local node join to topology event. */
-    private GridFutureAdapter<DiscoveryEvent> locJoinEvt = new GridFutureAdapter<>();
+    private GridFutureAdapter<T2<DiscoveryEvent, DiscoCache>> locJoin = new GridFutureAdapter<>();
 
     /** GC CPU load. */
     private volatile double gcCpuLoad;
@@ -542,20 +539,25 @@ private void updateClientNodes(UUID leftNodeId) {
                     }
                 }
 
+                final DiscoCache discoCache;
+
                 // Put topology snapshot into discovery history.
                 // There is no race possible between history maintenance and concurrent discovery
                 // event notifications, since SPI notifies manager about all events from this listener.
                 if (verChanged) {
-                    DiscoCache cache = new DiscoCache(locNode, F.view(topSnapshot, F.remoteNodes(locNode.id())));
+                    discoCache = createDiscoCache(locNode, topSnapshot);
 
-                    discoCacheHist.put(nextTopVer, cache);
+                    discoCacheHist.put(nextTopVer, discoCache);
 
-                    boolean set = updateTopologyVersionIfGreater(nextTopVer, cache);
+                    boolean set = updateTopologyVersionIfGreater(nextTopVer, discoCache);
 
                     assert set || topVer == 0 : "Topology version has not been updated [this.topVer=" +
                         topSnap + ", topVer=" + topVer + ", node=" + node +
                         ", evt=" + U.gridEventName(type) + ']';
                 }
+                else
+                    // Current version.
+                    discoCache = discoCache();
 
                 // If this is a local join event, just save it and do not notify listeners.
                 if (type == EVT_NODE_JOINED && node.id().equals(locNode.id())) {
@@ -563,7 +565,7 @@ private void updateClientNodes(UUID leftNodeId) {
                         gridStartTime = getSpi().getGridStartTime();
 
                     updateTopologyVersionIfGreater(new AffinityTopologyVersion(locNode.order()),
-                        new DiscoCache(localNode(), F.view(topSnapshot, F.remoteNodes(locNode.id()))));
+                        discoCache);
 
                     startLatch.countDown();
 
@@ -573,14 +575,9 @@ private void updateClientNodes(UUID leftNodeId) {
                     discoEvt.eventNode(node);
                     discoEvt.type(EVT_NODE_JOINED);
 
-                    discoEvt.topologySnapshot(topVer, new ArrayList<>(
-                        F.viewReadOnly(topSnapshot, new C1<ClusterNode, ClusterNode>() {
-                            @Override public ClusterNode apply(ClusterNode e) {
-                                return e;
-                            }
-                        }, FILTER_DAEMON)));
+                    discoEvt.topologySnapshot(topVer, new ArrayList<>(F.view(topSnapshot, FILTER_DAEMON)));
 
-                    locJoinEvt.onDone(discoEvt);
+                    locJoin.onDone(new T2<>(discoEvt, discoCache));
 
                     return;
                 }
@@ -595,7 +592,7 @@ else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
 
                     ((IgniteKernal)ctx.grid()).onDisconnected();
 
-                    locJoinEvt = new GridFutureAdapter<>();
+                    locJoin = new GridFutureAdapter<>();
 
                     registeredCaches.clear();
 
@@ -608,7 +605,7 @@ else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
                     topHist.clear();
 
                     topSnap.set(new Snapshot(AffinityTopologyVersion.ZERO,
-                        new DiscoCache(locNode, Collections.<ClusterNode>emptySet())));
+                        createDiscoCache(locNode, Collections.<ClusterNode>emptySet())));
                 }
                 else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                     assert locNode.isClient() : locNode;
@@ -625,7 +622,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                             try {
                                 fut.get();
 
-                                discoWrk.addEvent(type, nextTopVer, node, topSnapshot, null);
+                                discoWrk.addEvent(type, nextTopVer, node, discoCache, topSnapshot, null);
                             }
                             catch (IgniteException ignore) {
                                 // No-op.
@@ -637,7 +634,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                 }
 
                 if (type == EVT_CLIENT_NODE_DISCONNECTED || type == EVT_NODE_SEGMENTED || !ctx.clientDisconnected())
-                    discoWrk.addEvent(type, nextTopVer, node, topSnapshot, customMsg);
+                    discoWrk.addEvent(type, nextTopVer, node, discoCache, topSnapshot, customMsg);
             }
         });
 
@@ -1333,8 +1330,8 @@ private String topologySnapshotMessage(long topVer, int srvNodesNum, int clientN
             U.join(segChkThread, log);
         }
 
-        if (!locJoinEvt.isDone())
-            locJoinEvt.onDone(
+        if (!locJoin.isDone())
+            locJoin.onDone(
                 new IgniteCheckedException("Failed to wait for local node joined event (grid is stopping)."));
     }
 
@@ -1528,7 +1525,7 @@ public IgniteInternalFuture<Long> topologyFuture(final long awaitVer) {
      *
      * @return Discovery collection cache.
      */
-    private DiscoCache discoCache() {
+    public DiscoCache discoCache() {
         Snapshot cur = topSnap.get();
 
         assert cur != null;
@@ -1577,7 +1574,7 @@ public Collection<ClusterNode> nodes(long topVer) {
      * @return Collection of cache nodes.
      */
     public Collection<ClusterNode> nodes(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).allNodes();
+        return resolveDiscoCache(CU.cacheId(null), topVer).allNodes();
     }
 
     /**
@@ -1585,7 +1582,7 @@ public Collection<ClusterNode> nodes(AffinityTopologyVersion topVer) {
      * @return All server nodes for given topology version.
      */
     public List<ClusterNode> serverNodes(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).srvNodes;
+        return resolveDiscoCache(CU.cacheId(null), topVer).serverNodes();
     }
 
     /**
@@ -1596,7 +1593,7 @@ public List<ClusterNode> serverNodes(AffinityTopologyVersion topVer) {
      * @return Node.
      */
     public ClusterNode node(AffinityTopologyVersion topVer, UUID id) {
-        return resolveDiscoCache(null, topVer).node(id);
+        return resolveDiscoCache(CU.cacheId(null), topVer).node(id);
     }
 
     /**
@@ -1607,49 +1604,38 @@ public ClusterNode node(AffinityTopologyVersion topVer, UUID id) {
      * @return Collection of cache nodes.
      */
     public Collection<ClusterNode> cacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).cacheNodes(cacheName, topVer.topologyVersion());
+        return resolveDiscoCache(CU.cacheId(cacheName), topVer).cacheNodes(cacheName);
     }
 
     /**
-     * Gets all nodes with at least one cache configured.
+     * Gets cache nodes for cache with given ID.
      *
+     * @param cacheId Cache ID.
      * @param topVer Topology version.
      * @return Collection of cache nodes.
      */
-    public Collection<ClusterNode> cacheNodes(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).allNodesWithCaches(topVer.topologyVersion());
+    public Collection<ClusterNode> cacheNodes(int cacheId, AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(cacheId, topVer).cacheNodes(cacheId);
     }
 
     /**
-     * Gets cache remote nodes for cache with given name.
-     *
-     * @param topVer Topology version.
-     * @return Collection of cache nodes.
-     */
-    public Collection<ClusterNode> remoteCacheNodes(AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(null, topVer).remoteCacheNodes(topVer.topologyVersion());
-    }
-
-    /**
-     * Gets cache nodes for cache with given name.
+     * Gets all nodes with at least one cache configured.
      *
-     * @param cacheName Cache name.
      * @param topVer Topology version.
      * @return Collection of cache nodes.
      */
-    Collection<ClusterNode> aliveCacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).aliveCacheNodes(cacheName, topVer.topologyVersion());
+    public Collection<ClusterNode> cacheNodes(AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(CU.cacheId(null), topVer).allNodesWithCaches();
     }
 
     /**
      * Gets cache remote nodes for cache with given name.
      *
-     * @param cacheName Cache name.
      * @param topVer Topology version.
      * @return Collection of cache nodes.
      */
-    Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).aliveRemoteCacheNodes(cacheName, topVer.topologyVersion());
+    public Collection<ClusterNode> remoteCacheNodes(AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(CU.cacheId(null), topVer).remoteNodesWithCaches();
     }
 
     /**
@@ -1657,11 +1643,7 @@ Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, Affini
      * @return Oldest alive server nodes with at least one cache configured.
      */
     @Nullable public ClusterNode oldestAliveCacheServerNode(AffinityTopologyVersion topVer) {
-        DiscoCache cache = resolveDiscoCache(null, topVer);
-
-        Map.Entry<ClusterNode, Boolean> e = cache.aliveSrvNodesWithCaches.firstEntry();
-
-        return e != null ? e.getKey() : null;
+        return resolveDiscoCache(CU.cacheId(null), topVer).oldestAliveServerNodeWithCache();
     }
 
     /**
@@ -1672,7 +1654,20 @@ Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, Affini
      * @return Collection of cache affinity nodes.
      */
     public Collection<ClusterNode> cacheAffinityNodes(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).cacheAffinityNodes(cacheName, topVer.topologyVersion());
+        int cacheId = CU.cacheId(cacheName);
+
+        return resolveDiscoCache(cacheId, topVer).cacheAffinityNodes(cacheId);
+    }
+
+    /**
+     * Gets cache nodes for cache with given ID that participate in affinity calculation.
+     *
+     * @param cacheId Cache ID.
+     * @param topVer Topology version.
+     * @return Collection of cache affinity nodes.
+     */
+    public Collection<ClusterNode> cacheAffinityNodes(int cacheId, AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(cacheId, topVer).cacheAffinityNodes(cacheId);
     }
 
     /**
@@ -1742,31 +1737,34 @@ public Map<String, CacheMode> nodeCaches(ClusterNode node) {
     }
 
     /**
-     * Checks if cache with given name has at least one node with near cache enabled.
+     * Checks if cache with given ID has at least one node with near cache enabled.
      *
-     * @param cacheName Cache name.
+     * @param cacheId Cache ID.
      * @param topVer Topology version.
      * @return {@code True} if cache with given name has at least one node with near cache enabled.
      */
-    public boolean hasNearCache(@Nullable String cacheName, AffinityTopologyVersion topVer) {
-        return resolveDiscoCache(cacheName, topVer).hasNearCache(cacheName);
+    public boolean hasNearCache(int cacheId, AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(cacheId, topVer).hasNearCache(cacheId);
     }
 
     /**
      * Gets discovery cache for given topology version.
      *
-     * @param cacheName Cache name (participates in exception message).
+     * @param cacheId Cache ID (participates in exception message).
      * @param topVer Topology version.
      * @return Discovery cache.
      */
-    private DiscoCache resolveDiscoCache(@Nullable String cacheName, AffinityTopologyVersion topVer) {
+    private DiscoCache resolveDiscoCache(int cacheId, AffinityTopologyVersion topVer) {
         Snapshot snap = topSnap.get();
 
         DiscoCache cache = AffinityTopologyVersion.NONE.equals(topVer) || topVer.equals(snap.topVer) ?
             snap.discoCache : discoCacheHist.get(topVer);
 
         if (cache == null) {
-            throw new IgniteException("Failed to resolve nodes topology [cacheName=" + cacheName +
+            DynamicCacheDescriptor desc = ctx.cache().cacheDescriptor(cacheId);
+
+            throw new IgniteException("Failed to resolve nodes topology [" +
+                "cacheName=" + (desc != null ? desc.cacheConfiguration().getName() : "N/A") +
                 ", topVer=" + topVer +
                 ", history=" + discoCacheHist.keySet() +
                 ", snap=" + snap +
@@ -1817,7 +1815,19 @@ public AffinityTopologyVersion topologyVersionEx() {
     /** @return Event that represents a local node joined to topology. */
     public DiscoveryEvent localJoinEvent() {
         try {
-            return locJoinEvt.get();
+            return locJoin.get().get1();
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException(e);
+        }
+    }
+
+    /**
+     * @return Tuple that consists of a local join event and discovery cache at the join time.
+     */
+    public T2<DiscoveryEvent, DiscoCache> localJoin() {
+        try {
+            return locJoin.get();
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException(e);
@@ -1890,6 +1900,114 @@ public void failNode(UUID nodeId, @Nullable String warning) {
         }
     }
 
+    /**
+     * @param loc Local node.
+     * @param topSnapshot Topology snapshot.
+     * @return Newly created discovery cache.
+     */
+    @NotNull private DiscoCache createDiscoCache(ClusterNode loc, Collection<ClusterNode> topSnapshot) {
+        HashSet<UUID> alives = U.newHashSet(topSnapshot.size());
+        HashMap<UUID, ClusterNode> nodeMap = U.newHashMap(topSnapshot.size());
+
+        ArrayList<ClusterNode> daemonNodes = new ArrayList<>(topSnapshot.size());
+        ArrayList<ClusterNode> srvNodes = new ArrayList<>(topSnapshot.size());
+        ArrayList<ClusterNode> rmtNodes = new ArrayList<>(topSnapshot.size());
+        ArrayList<ClusterNode> allNodes = new ArrayList<>(topSnapshot.size());
+
+        for (ClusterNode node : topSnapshot) {
+            if (alive(node))
+                alives.add(node.id());
+
+            if (node.isDaemon())
+                daemonNodes.add(node);
+            else {
+                allNodes.add(node);
+
+                if (!node.isLocal())
+                    rmtNodes.add(node);
+
+                if (!CU.clientNode(node))
+                    srvNodes.add(node);
+            }
+
+            nodeMap.put(node.id(), node);
+        }
+
+        assert !rmtNodes.contains(loc) : "Remote nodes collection shouldn't contain local node" +
+            " [rmtNodes=" + rmtNodes + ", loc=" + loc + ']';
+
+        Map<Integer, List<ClusterNode>> allCacheNodes = U.newHashMap(allNodes.size());
+        Map<Integer, List<ClusterNode>> affCacheNodes = U.newHashMap(allNodes.size());
+
+        Set<ClusterNode> allNodesWithCaches = new TreeSet<>(GridNodeOrderComparator.INSTANCE);
+        Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(GridNodeOrderComparator.INSTANCE);
+        Set<ClusterNode> srvNodesWithCaches = new TreeSet<>(GridNodeOrderComparator.INSTANCE);
+
+        Set<Integer> nearEnabledCaches = new HashSet<>();
+
+        for (ClusterNode node : allNodes) {
+            assert node.order() != 0 : "Invalid node order [locNode=" + loc + ", node=" + node + ']';
+            assert !node.isDaemon();
+
+            for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
+                String cacheName = entry.getKey();
+                CachePredicate filter = entry.getValue();
+
+                if (filter.cacheNode(node)) {
+                    allNodesWithCaches.add(node);
+
+                    if(!CU.clientNode(node))
+                        srvNodesWithCaches.add(node);
+
+                    if (!node.isLocal())
+                        rmtNodesWithCaches.add(node);
+
+                    addToMap(allCacheNodes, cacheName, node);
+
+                    if (filter.dataNode(node))
+                        addToMap(affCacheNodes, cacheName, node);
+
+                    if (filter.nearNode(node))
+                        nearEnabledCaches.add(CU.cacheId(cacheName));
+                }
+            }
+        }
+
+        return new DiscoCache(
+            loc,
+            Collections.unmodifiableList(rmtNodes),
+            Collections.unmodifiableList(allNodes),
+            Collections.unmodifiableList(srvNodes),
+            Collections.unmodifiableList(daemonNodes),
+            U.sealList(srvNodesWithCaches),
+            U.sealList(allNodesWithCaches),
+            U.sealList(rmtNodesWithCaches),
+            Collections.unmodifiableMap(allCacheNodes),
+            Collections.unmodifiableMap(affCacheNodes),
+            Collections.unmodifiableMap(nodeMap),
+            Collections.unmodifiableSet(nearEnabledCaches),
+            alives);
+    }
+
+    /**
+     * Adds node to map.
+     *
+     * @param cacheMap Map to add to.
+     * @param cacheName Cache name.
+     * @param rich Node to add
+     */
+    private void addToMap(Map<Integer, List<ClusterNode>> cacheMap, String cacheName, ClusterNode rich) {
+        List<ClusterNode> cacheNodes = cacheMap.get(CU.cacheId(cacheName));
+
+        if (cacheNodes == null) {
+            cacheNodes = new ArrayList<>();
+
+            cacheMap.put(CU.cacheId(cacheName), cacheNodes);
+        }
+
+        cacheNodes.add(rich);
+    }
+
     /**
      * Updates topology version if current version is smaller than updated.
      *
@@ -1991,8 +2109,16 @@ public void scheduleSegmentCheck() {
                     lastChk = now;
 
                     if (!segValid) {
-                        discoWrk.addEvent(EVT_NODE_SEGMENTED, AffinityTopologyVersion.NONE, getSpi().getLocalNode(),
-                            Collections.<ClusterNode>emptyList(), null);
+                        List<ClusterNode> empty = Collections.emptyList();
+
+                        ClusterNode node = getSpi().getLocalNode();
+
+                        discoWrk.addEvent(EVT_NODE_SEGMENTED,
+                            AffinityTopologyVersion.NONE,
+                            node,
+                            createDiscoCache(node, empty),
+                            empty,
+                            null);
 
                         lastSegChkRes.set(false);
                     }
@@ -2012,8 +2138,8 @@ public void scheduleSegmentCheck() {
     /** Worker for discovery events. */
     private class DiscoveryWorker extends GridWorker {
         /** Event queue. */
-        private final BlockingQueue<GridTuple5<Integer, AffinityTopologyVersion, ClusterNode, Collection<ClusterNode>,
-            DiscoveryCustomMessage>> evts = new LinkedBlockingQueue<>();
+        private final BlockingQueue<GridTuple6<Integer, AffinityTopologyVersion, ClusterNode,
+            DiscoCache, Collection<ClusterNode>, DiscoveryCustomMessage>> evts = new LinkedBlockingQueue<>();
 
         /** Node segmented event fired flag. */
         private boolean nodeSegFired;
@@ -2031,10 +2157,11 @@ private DiscoveryWorker() {
          * @param type Discovery event type. See {@link DiscoveryEvent} for more details.
          * @param topVer Topology version.
          * @param node Remote node this event is connected with.
+         * @param discoCache Discovery cache.
          * @param topSnapshot Topology snapshot.
          */
         @SuppressWarnings("RedundantTypeArguments")
-        private void recordEvent(int type, long topVer, ClusterNode node, Collection<ClusterNode> topSnapshot) {
+        private void recordEvent(int type, long topVer, ClusterNode node, DiscoCache discoCache, Collection<ClusterNode> topSnapshot) {
             assert node != null;
 
             if (ctx.event().isRecordable(type)) {
@@ -2043,7 +2170,6 @@ private void recordEvent(int type, long topVer, ClusterNode node, Collection<Clu
                 evt.node(ctx.discovery().localNode());
                 evt.eventNode(node);
                 evt.type(type);
-
                 evt.topologySnapshot(topVer, U.<ClusterNode, ClusterNode>arrayList(topSnapshot, FILTER_DAEMON));
 
                 if (type == EVT_NODE_METRICS_UPDATED)
@@ -2070,7 +2196,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED)
                 else
                     assert false;
 
-                ctx.event().record(evt);
+                ctx.event().record(evt, discoCache);
             }
         }
 
@@ -2078,6 +2204,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED)
          * @param type Event type.
          * @param topVer Topology version.
          * @param node Node.
+         * @param discoCache Discovery cache.
          * @param topSnapshot Topology snapshot.
          * @param data Custom message.
          */
@@ -2085,12 +2212,13 @@ void addEvent(
             int type,
             AffinityTopologyVersion topVer,
             ClusterNode node,
+            DiscoCache discoCache,
             Collection<ClusterNode> topSnapshot,
             @Nullable DiscoveryCustomMessage data
         ) {
             assert node != null : data;
 
-            evts.add(new GridTuple5<>(type, topVer, node, topSnapshot, data));
+            evts.add(new GridTuple6<>(type, topVer, node, discoCache, topSnapshot, data));
         }
 
         /**
@@ -2127,7 +2255,7 @@ private String quietNode(ClusterNode node) {
         /** @throws InterruptedException If interrupted. */
         @SuppressWarnings("DuplicateCondition")
         private void body0() throws InterruptedException {
-            GridTuple5<Integer, AffinityTopologyVersion, ClusterNode, Collection<ClusterNode>,
+            GridTuple6<Integer, AffinityTopologyVersion, ClusterNode, DiscoCache, Collection<ClusterNode>,
                 DiscoveryCustomMessage> evt = evts.take();
 
             int type = evt.get1();
@@ -2260,11 +2388,11 @@ else if (log.isDebugEnabled())
                         customEvt.node(ctx.discovery().localNode());
                         customEvt.eventNode(node);
                         customEvt.type(type);
-                        customEvt.topologySnapshot(topVer.topologyVersion(), evt.get4());
+                        customEvt.topologySnapshot(topVer.topologyVersion(), evt.get5());
                         customEvt.affinityTopologyVersion(topVer);
-                        customEvt.customMessage(evt.get5());
+                        customEvt.customMessage(evt.get6());
 
-                        ctx.event().record(customEvt);
+                        ctx.event().record(customEvt, evt.get4());
                     }
 
                     return;
@@ -2278,7 +2406,7 @@ else if (log.isDebugEnabled())
                     assert false : "Invalid discovery event: " + type;
             }
 
-            recordEvent(type, topVer.topologyVersion(), node, evt.get4());
+            recordEvent(type, topVer.topologyVersion(), node, evt.get4(), evt.get5());
 
             if (segmented)
                 onSegmentation();
@@ -2488,432 +2616,6 @@ private Snapshot(AffinityTopologyVersion topVer, DiscoCache discoCache) {
         }
     }
 
-    /** Cache for discovery collections. */
-    private class DiscoCache {
-        /** Remote nodes. */
-        private final List<ClusterNode> rmtNodes;
-
-        /** All nodes. */
-        private final List<ClusterNode> allNodes;
-
-        /** All server nodes. */
-        private final List<ClusterNode> srvNodes;
-
-        /** All nodes with at least one cache configured. */
-        @GridToStringInclude
-        private final Collection<ClusterNode> allNodesWithCaches;
-
-        /** All nodes with at least one cache configured. */
-        @GridToStringInclude
-        private final Collection<ClusterNode> rmtNodesWithCaches;
-
-        /** Cache nodes by cache name. */
-        @GridToStringInclude
-        private final Map<String, Collection<ClusterNode>> allCacheNodes;
-
-        /** Remote cache nodes by cache name. */
-        @GridToStringInclude
-        private final Map<String, Collection<ClusterNode>> rmtCacheNodes;
-
-        /** Cache nodes by cache name. */
-        @GridToStringInclude
-        private final Map<String, Collection<ClusterNode>> affCacheNodes;
-
-        /** Caches where at least one node has near cache enabled. */
-        @GridToStringInclude
-        private final Set<String> nearEnabledCaches;
-
-        /** Nodes grouped by version. */
-        private final NavigableMap<IgniteProductVersion, Collection<ClusterNode>> nodesByVer;
-
-        /** Daemon nodes. */
-        private final List<ClusterNode> daemonNodes;
-
-        /** Node map. */
-        private final Map<UUID, ClusterNode> nodeMap;
-
-        /** Local node. */
-        private final ClusterNode loc;
-
-        /** Highest node order. */
-        private final long maxOrder;
-
-        /**
-         * Cached alive nodes list. As long as this collection doesn't accept {@code null}s use {@link
-         * #maskNull(String)} before passing raw cache names to it.
-         */
-        private final ConcurrentMap<String, Collection<ClusterNode>> aliveCacheNodes;
-
-        /**
-         * Cached alive remote nodes list. As long as this collection doesn't accept {@code null}s use {@link
-         * #maskNull(String)} before passing raw cache names to it.
-         */
-        private final ConcurrentMap<String, Collection<ClusterNode>> aliveRmtCacheNodes;
-
-        /**
-         * Cached alive server remote nodes with caches.
-         */
-        private final ConcurrentSkipListMap<ClusterNode, Boolean> aliveSrvNodesWithCaches;
-
-        /**
-         * @param loc Local node.
-         * @param rmts Remote nodes.
-         */
-        private DiscoCache(ClusterNode loc, Collection<ClusterNode> rmts) {
-            this.loc = loc;
-
-            rmtNodes = Collections.unmodifiableList(new ArrayList<>(F.view(rmts, FILTER_DAEMON)));
-
-            assert !rmtNodes.contains(loc) : "Remote nodes collection shouldn't contain local node" +
-                " [rmtNodes=" + rmtNodes + ", loc=" + loc + ']';
-
-            List<ClusterNode> all = new ArrayList<>(rmtNodes.size() + 1);
-
-            if (!loc.isDaemon())
-                all.add(loc);
-
-            all.addAll(rmtNodes);
-
-            Collections.sort(all, GridNodeOrderComparator.INSTANCE);
-
-            allNodes = Collections.unmodifiableList(all);
-
-            Map<String, Collection<ClusterNode>> cacheMap = new HashMap<>(allNodes.size(), 1.0f);
-            Map<String, Collection<ClusterNode>> rmtCacheMap = new HashMap<>(allNodes.size(), 1.0f);
-            Map<String, Collection<ClusterNode>> dhtNodesMap = new HashMap<>(allNodes.size(), 1.0f);
-            Collection<ClusterNode> nodesWithCaches = new HashSet<>(allNodes.size());
-            Collection<ClusterNode> rmtNodesWithCaches = new HashSet<>(allNodes.size());
-
-            aliveCacheNodes = new ConcurrentHashMap8<>(allNodes.size(), 1.0f);
-            aliveRmtCacheNodes = new ConcurrentHashMap8<>(allNodes.size(), 1.0f);
-            aliveSrvNodesWithCaches = new ConcurrentSkipListMap<>(GridNodeOrderComparator.INSTANCE);
-            nodesByVer = new TreeMap<>();
-
-            long maxOrder0 = 0;
-
-            Set<String> nearEnabledSet = new HashSet<>();
-
-            List<ClusterNode> srvNodes = new ArrayList<>();
-
-            for (ClusterNode node : allNodes) {
-                assert node.order() != 0 : "Invalid node order [locNode=" + loc + ", node=" + node + ']';
-                assert !node.isDaemon();
-
-                if (!CU.clientNode(node))
-                    srvNodes.add(node);
-
-                if (node.order() > maxOrder0)
-                    maxOrder0 = node.order();
-
-                boolean hasCaches = false;
-
-                for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
-                    String cacheName = entry.getKey();
-
-                    CachePredicate filter = entry.getValue();
-
-                    if (filter.cacheNode(node)) {
-                        nodesWithCaches.add(node);
-
-                        if (!loc.id().equals(node.id()))
-                            rmtNodesWithCaches.add(node);
-
-                        addToMap(cacheMap, cacheName, node);
-
-                        if (alive(node.id()))
-                            addToMap(aliveCacheNodes, maskNull(cacheName), node);
-
-                        if (filter.dataNode(node))
-                            addToMap(dhtNodesMap, cacheName, node);
-
-                        if (filter.nearNode(node))
-                            nearEnabledSet.add(cacheName);
-
-                        if (!loc.id().equals(node.id())) {
-                            addToMap(rmtCacheMap, cacheName, node);
-
-                            if (alive(node.id()))
-                                addToMap(aliveRmtCacheNodes, maskNull(cacheName), node);
-                        }
-
-                        hasCaches = true;
-                    }
-                }
-
-                if (hasCaches && alive(node.id()) && !CU.clientNode(node))
-                    aliveSrvNodesWithCaches.put(node, Boolean.TRUE);
-
-                IgniteProductVersion nodeVer = U.productVersion(node);
-
-                // Create collection for this version if it does not exist.
-                Collection<ClusterNode> nodes = nodesByVer.get(nodeVer);
-
-                if (nodes == null) {
-                    nodes = new ArrayList<>(allNodes.size());
-
-                    nodesByVer.put(nodeVer, nodes);
-                }
-
-                nodes.add(node);
-            }
-
-            Collections.sort(srvNodes, CU.nodeComparator(true));
-
-            // Need second iteration to add this node to all previous node versions.
-            for (ClusterNode node : allNodes) {
-                IgniteProductVersion nodeVer = U.productVersion(node);
-
-                // Get all versions lower or equal node's version.
-                NavigableMap<IgniteProductVersion, Collection<ClusterNode>> updateView =
-                    nodesByVer.headMap(nodeVer, false);
-
-                for (Collection<ClusterNode> prevVersions : updateView.values())
-                    prevVersions.add(node);
-            }
-
-            maxOrder = maxOrder0;
-
-            allCacheNodes = Collections.unmodifiableMap(cacheMap);
-            rmtCacheNodes = Collections.unmodifiableMap(rmtCacheMap);
-            affCacheNodes = Collections.unmodifiableMap(dhtNodesMap);
-            allNodesWithCaches = Collections.unmodifiableCollection(nodesWithCaches);
-            this.rmtNodesWithCaches = Collections.unmodifiableCollection(rmtNodesWithCaches);
-            nearEnabledCaches = Collections.unmodifiableSet(nearEnabledSet);
-            this.srvNodes = Collections.unmodifiableList(srvNodes);
-
-            daemonNodes = Collections.unmodifiableList(new ArrayList<>(
-                F.view(F.concat(false, loc, rmts), F0.not(FILTER_DAEMON))));
-
-            Map<UUID, ClusterNode> nodeMap = new HashMap<>(allNodes().size() + daemonNodes.size(), 1.0f);
-
-            for (ClusterNode n : F.concat(false, allNodes(), daemonNodes()))
-                nodeMap.put(n.id(), n);
-
-            this.nodeMap = nodeMap;
-        }
-
-        /**
-         * Adds node to map.
-         *
-         * @param cacheMap Map to add to.
-         * @param cacheName Cache name.
-         * @param rich Node to add
-         */
-        private void addToMap(Map<String, Collection<ClusterNode>> cacheMap, String cacheName, ClusterNode rich) {
-            Collection<ClusterNode> cacheNodes = cacheMap.get(cacheName);
-
-            if (cacheNodes == null) {
-                cacheNodes = new ArrayList<>(allNodes.size());
-
-                cacheMap.put(cacheName, cacheNodes);
-            }
-
-            cacheNodes.add(rich);
-        }
-
-        /** @return Local node. */
-        ClusterNode localNode() {
-            return loc;
-        }
-
-        /** @return Remote nodes. */
-        Collection<ClusterNode> remoteNodes() {
-            return rmtNodes;
-        }
-
-        /** @return All nodes. */
-        Collection<ClusterNode> allNodes() {
-            return allNodes;
-        }
-
-        /**
-         * Gets collection of nodes which have version equal or greater than {@code ver}.
-         *
-         * @param ver Version to check.
-         * @return Collection of nodes with version equal or greater than {@code ver}.
-         */
-        Collection<ClusterNode> elderNodes(IgniteProductVersion ver) {
-            Map.Entry<IgniteProductVersion, Collection<ClusterNode>> entry = nodesByVer.ceilingEntry(ver);
-
-            if (entry == null)
-                return Collections.emptyList();
-
-            return entry.getValue();
-        }
-
-        /**
-         * @return Versions map.
-         */
-        NavigableMap<IgniteProductVersion, Collection<ClusterNode>> versionsMap() {
-            return nodesByVer;
-        }
-
-        /**
-         * Gets collection of nodes with at least one cache configured.
-         *
-         * @param topVer Topology version (maximum allowed node order).
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> allNodesWithCaches(final long topVer) {
-            return filter(topVer, allNodesWithCaches);
-        }
-
-        /**
-         * Gets all nodes that have cache with given name.
-         *
-         * @param cacheName Cache name.
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> cacheNodes(@Nullable String cacheName, final long topVer) {
-            return filter(topVer, allCacheNodes.get(cacheName));
-        }
-
-        /**
-         * Gets all remote nodes that have at least one cache configured.
-         *
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> remoteCacheNodes(final long topVer) {
-            return filter(topVer, rmtNodesWithCaches);
-        }
-
-        /**
-         * Gets all nodes that have cache with given name and should participate in affinity calculation. With
-         * partitioned cache nodes with near-only cache do not participate in affinity node calculation.
-         *
-         * @param cacheName Cache name.
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> cacheAffinityNodes(@Nullable String cacheName, final long topVer) {
-            return filter(topVer, affCacheNodes.get(cacheName));
-        }
-
-        /**
-         * Gets all alive nodes that have cache with given name.
-         *
-         * @param cacheName Cache name.
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> aliveCacheNodes(@Nullable String cacheName, final long topVer) {
-            return filter(topVer, aliveCacheNodes.get(maskNull(cacheName)));
-        }
-
-        /**
-         * Gets all alive remote nodes that have cache with given name.
-         *
-         * @param cacheName Cache name.
-         * @param topVer Topology version.
-         * @return Collection of nodes.
-         */
-        Collection<ClusterNode> aliveRemoteCacheNodes(@Nullable String cacheName, final long topVer) {
-            return filter(topVer, aliveRmtCacheNodes.get(maskNull(cacheName)));
-        }
-
-        /**
-         * Checks if cache with given name has at least one node with near cache enabled.
-         *
-         * @param cacheName Cache name.
-         * @return {@code True} if cache with given name has at least one node with near cache enabled.
-         */
-        boolean hasNearCache(@Nullable String cacheName) {
-            return nearEnabledCaches.contains(cacheName);
-        }
-
-        /**
-         * Removes left node from cached alives lists.
-         *
-         * @param leftNode Left node.
-         */
-        void updateAlives(ClusterNode leftNode) {
-            if (leftNode.order() > maxOrder)
-                return;
-
-            filterNodeMap(aliveCacheNodes, leftNode);
-
-            filterNodeMap(aliveRmtCacheNodes, leftNode);
-
-            aliveSrvNodesWithCaches.remove(leftNode);
-        }
-
-        /**
-         * Creates a copy of nodes map without the given node.
-         *
-         * @param map Map to copy.
-         * @param exclNode Node to exclude.
-         */
-        private void filterNodeMap(ConcurrentMap<String, Collection<ClusterNode>> map, final ClusterNode exclNode) {
-            for (String cacheName : registeredCaches.keySet()) {
-                String maskedName = maskNull(cacheName);
-
-                while (true) {
-                    Collection<ClusterNode> oldNodes = map.get(maskedName);
-
-                    if (oldNodes == null || oldNodes.isEmpty())
-                        break;
-
-                    Collection<ClusterNode> newNodes = new ArrayList<>(oldNodes);
-
-                    if (!newNodes.remove(exclNode))
-                        break;
-
-                    if (map.replace(maskedName, oldNodes, newNodes))
-                        break;
-                }
-            }
-        }
-
-        /**
-         * Replaces {@code null} with {@code NULL_CACHE_NAME}.
-         *
-         * @param cacheName Cache name.
-         * @return Masked name.
-         */
-        private String maskNull(@Nullable String cacheName) {
-            return cacheName == null ? NULL_CACHE_NAME : cacheName;
-        }
-
-        /**
-         * @param topVer Topology version.
-         * @param nodes Nodes.
-         * @return Filtered collection (potentially empty, but never {@code null}).
-         */
-        private Collection<ClusterNode> filter(final long topVer, @Nullable Collection<ClusterNode> nodes) {
-            if (nodes == null)
-                return Collections.emptyList();
-
-            // If no filtering needed, return original collection.
-            return nodes.isEmpty() || topVer < 0 || topVer >= maxOrder ?
-                nodes :
-                F.view(nodes, new P1<ClusterNode>() {
-                    @Override public boolean apply(ClusterNode node) {
-                        return node.order() <= topVer;
-                    }
-                });
-        }
-
-        /** @return Daemon nodes. */
-        Collection<ClusterNode> daemonNodes() {
-            return daemonNodes;
-        }
-
-        /**
-         * @param id Node ID.
-         * @return Node.
-         */
-        @Nullable ClusterNode node(UUID id) {
-            return nodeMap.get(id);
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return S.toString(DiscoCache.class, this, "allNodesWithDaemons", U.toShortString(allNodes));
-        }
-    }
-
     /**
      * Cache predicate.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/DiscoveryEventListener.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/DiscoveryEventListener.java
new file mode 100644
index 0000000000000..963d97eefee84
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/DiscoveryEventListener.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.eventstorage;
+
+import java.util.EventListener;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+
+/**
+ * Internal listener for discovery events.
+ */
+public interface DiscoveryEventListener extends EventListener {
+    /**
+     * @param evt Discovery event.
+     * @param discoCache Discovery cache.
+     */
+    public void onEvent(DiscoveryEvent evt, DiscoCache discoCache);
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
index 607bb9688aaf1..a2c64ba33d8b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
@@ -47,6 +47,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoManager;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.deployment.GridDeployment;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.platform.PlatformEventFilterListener;
 import org.apache.ignite.internal.util.GridConcurrentLinkedHashSet;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -79,6 +80,9 @@ public class GridEventStorageManager extends GridManagerAdapter<EventStorageSpi>
     /** Local event listeners. */
     private final ConcurrentMap<Integer, Set<GridLocalEventListener>> lsnrs = new ConcurrentHashMap8<>();
 
+    /** Internal discovery listeners. */
+    private final ConcurrentMap<Integer, Set<DiscoveryEventListener>> discoLsnrs = new ConcurrentHashMap8<>();
+
     /** Busy lock to control activity of threads. */
     private final ReadWriteLock busyLock = new ReentrantReadWriteLock();
 
@@ -234,6 +238,7 @@ private void leaveBusy() {
         msgLsnr = null;
 
         lsnrs.clear();
+        discoLsnrs.clear();
     }
 
     /** {@inheritDoc} */
@@ -299,6 +304,30 @@ public void record(Event evt) {
         }
     }
 
+    /**
+     * Records discovery events.
+     *
+     * @param evt Event to record.
+     * @param discoCache Discovery cache.
+     */
+    public void record(DiscoveryEvent evt, DiscoCache discoCache) {
+        assert evt != null;
+
+        if (!enterBusy())
+            return;
+
+        try {
+            // Notify internal discovery listeners first.
+            notifyDiscoveryListeners(evt, discoCache);
+
+            // Notify all other registered listeners.
+            record(evt);
+        }
+        finally {
+            leaveBusy();
+        }
+    }
+
     /**
      * Gets types of enabled user-recordable events.
      *
@@ -570,7 +599,7 @@ public void addLocalEventListener(GridLocalEventListener lsnr, int[] types) {
 
         try {
             for (int t : types) {
-                getOrCreate(t).add(lsnr);
+                getOrCreate(lsnrs, t).add(lsnr);
 
                 if (!isRecordable(t))
                     U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
@@ -595,14 +624,14 @@ public void addLocalEventListener(GridLocalEventListener lsnr, int type, @Nullab
             return;
 
         try {
-            getOrCreate(type).add(lsnr);
+            getOrCreate(lsnrs, type).add(lsnr);
 
             if (!isRecordable(type))
                 U.warn(log, "Added listener for disabled event type: " + U.gridEventName(type));
 
             if (types != null) {
                 for (int t : types) {
-                    getOrCreate(t).add(lsnr);
+                    getOrCreate(lsnrs, t).add(lsnr);
 
                     if (!isRecordable(t))
                         U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
@@ -615,16 +644,70 @@ public void addLocalEventListener(GridLocalEventListener lsnr, int type, @Nullab
     }
 
     /**
+     * Adds discovery event listener. Note that this method specifically disallow an empty
+     * array of event type to prevent accidental subscription for all system event that
+     * may lead to a drastic performance decrease.
+     *
+     * @param lsnr Listener to add.
+     * @param types Event types to subscribe listener for.
+     */
+    public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int[] types) {
+        assert lsnr != null;
+        assert types != null;
+        assert types.length > 0;
+
+        if (!enterBusy())
+            return;
+
+        try {
+            for (int t : types) {
+                getOrCreate(discoLsnrs, t).add(lsnr);
+            }
+        }
+        finally {
+            leaveBusy();
+        }
+    }
+
+    /**
+     * Adds discovery event listener.
+     *
+     * @param lsnr Listener to add.
+     * @param type Event type to subscribe listener for.
+     * @param types Additional event types to subscribe listener for.
+     */
+    public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int type, @Nullable int... types) {
+        assert lsnr != null;
+
+        if (!enterBusy())
+            return;
+
+        try {
+            getOrCreate(discoLsnrs, type).add(lsnr);
+
+            if (types != null) {
+                for (int t : types) {
+                    getOrCreate(discoLsnrs, t).add(lsnr);
+                }
+            }
+        }
+        finally {
+            leaveBusy();
+        }
+    }
+
+    /**
+     * @param lsnrs Listeners map.
      * @param type Event type.
      * @return Listeners for given event type.
      */
-    private Collection<GridLocalEventListener> getOrCreate(Integer type) {
-        Set<GridLocalEventListener> set = lsnrs.get(type);
+    private <T> Collection<T> getOrCreate(ConcurrentMap<Integer, Set<T>> lsnrs, Integer type) {
+        Set<T> set = lsnrs.get(type);
 
         if (set == null) {
             set = new GridConcurrentLinkedHashSet<>();
 
-            Set<GridLocalEventListener> prev = lsnrs.putIfAbsent(type, set);
+            Set<T> prev = lsnrs.putIfAbsent(type, set);
 
             if (prev != null)
                 set = prev;
@@ -687,6 +770,38 @@ public boolean removeLocalEventListener(GridLocalEventListener lsnr, @Nullable i
         return found;
     }
 
+    /**
+     * Removes listener for specified events, if any. If no event types provided - it
+     * remove the listener for all its registered events.
+     *
+     * @param lsnr Listener.
+     * @param types Event types.
+     * @return Returns {@code true} if removed.
+     */
+    public boolean removeDiscoveryEventListener(DiscoveryEventListener lsnr, @Nullable int... types) {
+        assert lsnr != null;
+
+        boolean found = false;
+
+        if (F.isEmpty(types)) {
+            for (Set<DiscoveryEventListener> set : discoLsnrs.values())
+                if (set.remove(lsnr))
+                    found = true;
+        }
+        else {
+            assert types != null;
+
+            for (int type : types) {
+                Set<DiscoveryEventListener> set = discoLsnrs.get(type);
+
+                if (set != null && set.remove(lsnr))
+                    found = true;
+            }
+        }
+
+        return found;
+    }
+
     /**
      *
      * @param p Optional predicate.
@@ -779,6 +894,41 @@ private void notifyListeners(@Nullable Collection<GridLocalEventListener> set, E
         }
     }
 
+    /**
+     * @param evt Discovery event
+     * @param cache Discovery cache.
+     */
+    private void notifyDiscoveryListeners(DiscoveryEvent evt, DiscoCache cache) {
+        assert evt != null;
+
+        notifyDiscoveryListeners(discoLsnrs.get(evt.type()), evt, cache);
+    }
+
+    /**
+     * @param set Set of listeners.
+     * @param evt Discovery event.
+     * @param cache Discovery cache.
+     */
+    private void notifyDiscoveryListeners(@Nullable Collection<DiscoveryEventListener> set, DiscoveryEvent evt, DiscoCache cache) {
+        assert evt != null;
+
+        if (!F.isEmpty(set)) {
+            assert set != null;
+
+            for (DiscoveryEventListener lsnr : set) {
+                try {
+                    lsnr.onEvent(evt, cache);
+                }
+                catch (Throwable e) {
+                    U.error(log, "Unexpected exception in listener notification for event: " + evt, e);
+
+                    if (e instanceof Error)
+                        throw (Error)e;
+                }
+            }
+        }
+    }
+
     /**
      * @param p Grid event predicate.
      * @return Collection of grid events.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index a388c7affb65a..50704622017df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -252,10 +253,12 @@ public void onReconnected() {
      *
      * @param topVer Topology version to calculate affinity cache for.
      * @param discoEvt Discovery event that caused this topology version change.
+     * @param discoCache Discovery cache.
      * @return Affinity assignments.
      */
     @SuppressWarnings("IfMayBeConditional")
-    public List<List<ClusterNode>> calculate(AffinityTopologyVersion topVer, DiscoveryEvent discoEvt) {
+    public List<List<ClusterNode>> calculate(AffinityTopologyVersion topVer, DiscoveryEvent discoEvt,
+        DiscoCache discoCache) {
         if (log.isDebugEnabled())
             log.debug("Calculating affinity [topVer=" + topVer + ", locNodeId=" + ctx.localNodeId() +
                 ", discoEvt=" + discoEvt + ']');
@@ -266,7 +269,7 @@ public List<List<ClusterNode>> calculate(AffinityTopologyVersion topVer, Discove
         List<ClusterNode> sorted;
 
         if (!locCache) {
-            sorted = new ArrayList<>(ctx.discovery().cacheAffinityNodes(cacheName, topVer));
+            sorted = new ArrayList<>(discoCache.cacheAffinityNodes(cacheId()));
 
             Collections.sort(sorted, GridNodeOrderComparator.INSTANCE);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 2890887cca416..2642d16c56fc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -382,7 +382,7 @@ else if (req.start() && !req.clientStartOnly()) {
                 cctx.cache().prepareCacheStart(req, fut.topologyVersion());
 
                 if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
-                    if (cctx.discovery().cacheAffinityNodes(req.cacheName(), fut.topologyVersion()).isEmpty())
+                    if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
                         U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
                 }
 
@@ -403,7 +403,7 @@ else if (!req.clientStartOnly()) {
                             assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
 
                             List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
-                                fut.discoveryEvent());
+                                fut.discoveryEvent(), fut.discoCache());
 
                             aff.initialize(fut.topologyVersion(), assignment);
                         }
@@ -753,7 +753,7 @@ private void initStartedCacheOnCoordinator(GridDhtPartitionsExchangeFuture fut,
 
             assert old == null : old;
 
-            List<List<ClusterNode>> newAff = cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent());
+            List<List<ClusterNode>> newAff = cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
             cache.affinity().initialize(fut.topologyVersion(), newAff);
         }
@@ -791,7 +791,7 @@ public void initStartedCaches(boolean crd,
 
                     if (cache.affinity().lastVersion().equals(AffinityTopologyVersion.NONE)) {
                         List<List<ClusterNode>> assignment =
-                            cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent());
+                            cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
                         cache.affinity().initialize(fut.topologyVersion(), assignment);
                     }
@@ -817,14 +817,15 @@ public void initStartedCaches(boolean crd,
     private void initAffinity(GridAffinityAssignmentCache aff, GridDhtPartitionsExchangeFuture fut, boolean fetch)
         throws IgniteCheckedException {
         if (!fetch && canCalculateAffinity(aff, fut)) {
-            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(), fut.discoveryEvent());
+            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
             aff.initialize(fut.topologyVersion(), assignment);
         }
         else {
             GridDhtAssignmentFetchFuture fetchFut = new GridDhtAssignmentFetchFuture(cctx,
                 aff.cacheName(),
-                fut.topologyVersion());
+                fut.topologyVersion(),
+                fut.discoCache());
 
             fetchFut.init();
 
@@ -878,7 +879,7 @@ public void onServerJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd)
 
                             CacheHolder cache = cache(fut, cacheDesc);
 
-                            List<List<ClusterNode>> newAff = cache.affinity().calculate(topVer, fut.discoveryEvent());
+                            List<List<ClusterNode>> newAff = cache.affinity().calculate(topVer, fut.discoveryEvent(), fut.discoCache());
 
                             cache.affinity().initialize(topVer, newAff);
                         }
@@ -945,14 +946,15 @@ private void fetchAffinityOnJoin(GridDhtPartitionsExchangeFuture fut) throws Ign
 
             if (cctx.localNodeId().equals(cacheDesc.receivedFrom())) {
                 List<List<ClusterNode>> assignment =
-                    cacheCtx.affinity().affinityCache().calculate(fut.topologyVersion(), fut.discoveryEvent());
+                    cacheCtx.affinity().affinityCache().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
                 cacheCtx.affinity().affinityCache().initialize(fut.topologyVersion(), assignment);
             }
             else {
                 GridDhtAssignmentFetchFuture fetchFut = new GridDhtAssignmentFetchFuture(cctx,
                     cacheCtx.name(),
-                    topVer);
+                    topVer,
+                    fut.discoCache());
 
                 fetchFut.init();
 
@@ -986,7 +988,7 @@ private void fetchAffinity(GridDhtPartitionsExchangeFuture fut,
         GridDhtAffinityAssignmentResponse res = fetchFut.get();
 
         if (res == null) {
-            List<List<ClusterNode>> aff = affCache.calculate(topVer, fut.discoveryEvent());
+            List<List<ClusterNode>> aff = affCache.calculate(topVer, fut.discoveryEvent(), fut.discoCache());
 
             affCache.initialize(topVer, aff);
         }
@@ -998,7 +1000,7 @@ private void fetchAffinity(GridDhtPartitionsExchangeFuture fut,
             else {
                 assert !affCache.centralizedAffinityFunction() || !lateAffAssign;
 
-                affCache.calculate(topVer, fut.discoveryEvent());
+                affCache.calculate(topVer, fut.discoveryEvent(), fut.discoCache());
             }
 
             List<List<ClusterNode>> aff = res.affinityAssignment(cctx.discovery());
@@ -1028,7 +1030,7 @@ public boolean onServerLeft(final GridDhtPartitionsExchangeFuture fut) throws Ig
                 if (cacheCtx.isLocal())
                     continue;
 
-                cacheCtx.affinity().affinityCache().calculate(fut.topologyVersion(), fut.discoveryEvent());
+                cacheCtx.affinity().affinityCache().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
             }
 
             centralizedAff = true;
@@ -1078,7 +1080,7 @@ private IgniteInternalFuture<?> initCoordinatorCaches(final GridDhtPartitionsExc
 
                 if (cache != null) {
                     if (cache.client())
-                        cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent());
+                        cache.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
                     return;
                 }
@@ -1118,7 +1120,8 @@ private IgniteInternalFuture<?> initCoordinatorCaches(final GridDhtPartitionsExc
 
                     GridDhtAssignmentFetchFuture fetchFut = new GridDhtAssignmentFetchFuture(cctx,
                         aff.cacheName(),
-                        prev.topologyVersion());
+                        prev.topologyVersion(),
+                        prev.discoCache());
 
                     fetchFut.init();
 
@@ -1129,7 +1132,7 @@ private IgniteInternalFuture<?> initCoordinatorCaches(final GridDhtPartitionsExc
                             throws IgniteCheckedException {
                             fetchAffinity(prev, aff, (GridDhtAssignmentFetchFuture)fetchFut);
 
-                            aff.calculate(fut.topologyVersion(), fut.discoveryEvent());
+                            aff.calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache());
 
                             affFut.onDone(fut.topologyVersion());
                         }
@@ -1269,7 +1272,7 @@ private void initAffinityOnNodeJoin(GridDhtPartitionsExchangeFuture fut,
 
         assert aff.idealAssignment() != null : "Previous assignment is not available.";
 
-        List<List<ClusterNode>> idealAssignment = aff.calculate(topVer, fut.discoveryEvent());
+        List<List<ClusterNode>> idealAssignment = aff.calculate(topVer, fut.discoveryEvent(), fut.discoCache());
         List<List<ClusterNode>> newAssignment = null;
 
         if (latePrimary) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
index 8b7be1b3288e2..d9ff61684e7d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
@@ -77,7 +77,7 @@ public class GridCacheAffinityManager extends GridCacheManagerAdapter {
     @Override protected void onKernalStart0() throws IgniteCheckedException {
         if (cctx.isLocal())
             // No discovery event needed for local affinity.
-            aff.calculate(LOC_CACHE_TOP_VER, null);
+            aff.calculate(LOC_CACHE_TOP_VER, null, null);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7cf75fee86be3..81f21a8405fb3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -48,14 +48,14 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
-import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
-import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
@@ -173,35 +173,33 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     private DateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS");
 
     /** Discovery listener. */
-    private final GridLocalEventListener discoLsnr = new GridLocalEventListener() {
-        @Override public void onEvent(Event evt) {
+    private final DiscoveryEventListener discoLsnr = new DiscoveryEventListener() {
+        @Override public void onEvent(DiscoveryEvent evt, DiscoCache cache) {
             if (!enterBusy())
                 return;
 
             try {
-                DiscoveryEvent e = (DiscoveryEvent)evt;
-
                 ClusterNode loc = cctx.localNode();
 
-                assert e.type() == EVT_NODE_JOINED || e.type() == EVT_NODE_LEFT || e.type() == EVT_NODE_FAILED ||
-                    e.type() == EVT_DISCOVERY_CUSTOM_EVT;
+                assert evt.type() == EVT_NODE_JOINED || evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED ||
+                    evt.type() == EVT_DISCOVERY_CUSTOM_EVT;
 
-                final ClusterNode n = e.eventNode();
+                final ClusterNode n = evt.eventNode();
 
                 GridDhtPartitionExchangeId exchId = null;
                 GridDhtPartitionsExchangeFuture exchFut = null;
 
-                if (e.type() != EVT_DISCOVERY_CUSTOM_EVT) {
+                if (evt.type() != EVT_DISCOVERY_CUSTOM_EVT) {
                     assert !loc.id().equals(n.id());
 
-                    if (e.type() == EVT_NODE_LEFT || e.type() == EVT_NODE_FAILED) {
+                    if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) {
                         assert cctx.discovery().node(n.id()) == null;
 
                         // Avoid race b/w initial future add and discovery event.
                         GridDhtPartitionsExchangeFuture initFut = null;
 
                         if (readyTopVer.get().equals(AffinityTopologyVersion.NONE)) {
-                            initFut = exchangeFuture(initialExchangeId(), null, null, null);
+                            initFut = exchangeFuture(initialExchangeId(), null, null, null, null);
 
                             initFut.onNodeLeft(n);
                         }
@@ -213,18 +211,18 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                     }
 
                     assert
-                        e.type() != EVT_NODE_JOINED || n.order() > loc.order() :
+                        evt.type() != EVT_NODE_JOINED || n.order() > loc.order() :
                         "Node joined with smaller-than-local " +
                             "order [newOrder=" + n.order() + ", locOrder=" + loc.order() + ']';
 
                     exchId = exchangeId(n.id(),
-                        affinityTopologyVersion(e),
-                        e.type());
+                        affinityTopologyVersion(evt),
+                        evt.type());
 
-                    exchFut = exchangeFuture(exchId, e, null, null);
+                    exchFut = exchangeFuture(exchId, evt, cache,null, null);
                 }
                 else {
-                    DiscoveryCustomEvent customEvt = (DiscoveryCustomEvent)e;
+                    DiscoveryCustomEvent customEvt = (DiscoveryCustomEvent)evt;
 
                     if (customEvt.customMessage() instanceof DynamicCacheChangeBatch) {
                         DynamicCacheChangeBatch batch = (DynamicCacheChangeBatch)customEvt.customMessage();
@@ -254,9 +252,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                         }
 
                         if (!F.isEmpty(valid)) {
-                            exchId = exchangeId(n.id(), affinityTopologyVersion(e), e.type());
+                            exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type());
 
-                            exchFut = exchangeFuture(exchId, e, valid, null);
+                            exchFut = exchangeFuture(exchId, evt, cache, valid, null);
                         }
                     }
                     else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
@@ -264,13 +262,13 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
 
                         if (msg.exchangeId() == null) {
                             if (msg.exchangeNeeded()) {
-                                exchId = exchangeId(n.id(), affinityTopologyVersion(e), e.type());
+                                exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type());
 
-                                exchFut = exchangeFuture(exchId, e, null, msg);
+                                exchFut = exchangeFuture(exchId, evt, cache, null, msg);
                             }
                         }
                         else
-                            exchangeFuture(msg.exchangeId(), null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
+                            exchangeFuture(msg.exchangeId(), null, null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
                     }
                 }
 
@@ -279,7 +277,7 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                         log.debug("Discovery event (will start exchange): " + exchId);
 
                     // Event callback - without this callback future will never complete.
-                    exchFut.onEvent(exchId, e);
+                    exchFut.onEvent(exchId, evt, cache);
 
                     // Start exchange process.
                     addFuture(exchFut);
@@ -301,7 +299,7 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
 
         exchWorker = new ExchangeWorker();
 
-        cctx.gridEvents().addLocalEventListener(discoLsnr, EVT_NODE_JOINED, EVT_NODE_LEFT, EVT_NODE_FAILED,
+        cctx.gridEvents().addDiscoveryEventListener(discoLsnr, EVT_NODE_JOINED, EVT_NODE_LEFT, EVT_NODE_FAILED,
             EVT_DISCOVERY_CUSTOM_EVT);
 
         cctx.io().addHandler(0, GridDhtPartitionsSingleMessage.class,
@@ -359,11 +357,14 @@ private GridDhtPartitionExchangeId initialExchangeId() {
         assert startTime > 0;
 
         // Generate dummy discovery event for local node joining.
-        DiscoveryEvent discoEvt = cctx.discovery().localJoinEvent();
+        T2<DiscoveryEvent, DiscoCache> localJoin = cctx.discovery().localJoin();
+
+        DiscoveryEvent discoEvt = localJoin.get1();
+        DiscoCache discoCache = localJoin.get2();
 
         GridDhtPartitionExchangeId exchId = initialExchangeId();
 
-        GridDhtPartitionsExchangeFuture fut = exchangeFuture(exchId, discoEvt, null, null);
+        GridDhtPartitionsExchangeFuture fut = exchangeFuture(exchId, discoEvt, discoCache, null, null);
 
         if (reconnect)
             reconnectExchangeFut = new GridFutureAdapter<>();
@@ -470,7 +471,7 @@ public static Object rebalanceTopic(int idx) {
 
     /** {@inheritDoc} */
     @Override protected void onKernalStop0(boolean cancel) {
-        cctx.gridEvents().removeLocalEventListener(discoLsnr);
+        cctx.gridEvents().removeDiscoveryEventListener(discoLsnr);
 
         cctx.io().removeHandler(0, GridDhtPartitionsSingleMessage.class);
         cctx.io().removeHandler(0, GridDhtPartitionsFullMessage.class);
@@ -1063,12 +1064,14 @@ private GridDhtPartitionExchangeId exchangeId(UUID nodeId, AffinityTopologyVersi
     /**
      * @param exchId Exchange ID.
      * @param discoEvt Discovery event.
+     * @param cache Discovery data cache.
      * @param reqs Cache change requests.
      * @param affChangeMsg Affinity change message.
      * @return Exchange future.
      */
     private GridDhtPartitionsExchangeFuture exchangeFuture(GridDhtPartitionExchangeId exchId,
         @Nullable DiscoveryEvent discoEvt,
+        @Nullable DiscoCache cache,
         @Nullable Collection<DynamicCacheChangeRequest> reqs,
         @Nullable CacheAffinityChangeMessage affChangeMsg) {
         GridDhtPartitionsExchangeFuture fut;
@@ -1087,7 +1090,7 @@ private GridDhtPartitionsExchangeFuture exchangeFuture(GridDhtPartitionExchangeI
         }
 
         if (discoEvt != null)
-            fut.onEvent(exchId, discoEvt);
+            fut.onEvent(exchId, discoEvt, cache);
 
         if (stopErr != null)
             fut.onDone(stopErr);
@@ -1231,7 +1234,7 @@ else if (!cacheCtx.isLocal())
                     refreshPartitions();
             }
             else
-                exchangeFuture(msg.exchangeId(), null, null, null).onReceive(node, msg);
+                exchangeFuture(msg.exchangeId(), null, null, null, null).onReceive(node, msg);
         }
         finally {
             leaveBusy();
@@ -1285,6 +1288,7 @@ else if (!cacheCtx.isLocal())
             else {
                 if (msg.client()) {
                     final GridDhtPartitionsExchangeFuture exchFut = exchangeFuture(msg.exchangeId(),
+                        null,
                         null,
                         null,
                         null);
@@ -1297,7 +1301,7 @@ else if (!cacheCtx.isLocal())
                     });
                 }
                 else
-                    exchangeFuture(msg.exchangeId(), null, null, null).onReceive(node, msg);
+                    exchangeFuture(msg.exchangeId(), null, null, null, null).onReceive(node, msg);
             }
         }
         finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 816132d351139..9366d0ca127c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
@@ -42,7 +43,6 @@
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
@@ -103,6 +103,9 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
     /** */
     private final Object similarAffKey;
 
+    /** */
+    private volatile DiscoCache discoCache;
+
     /**
      * @param cctx Context.
      * @param cacheId Cache ID.
@@ -121,6 +124,8 @@ public GridClientPartitionTopology(
 
         topVer = exchFut.topologyVersion();
 
+        discoCache = exchFut.discoCache();
+
         log = cctx.logger(getClass());
 
         lock.writeLock().lock();
@@ -191,6 +196,7 @@ public int cacheId() {
             this.stopping = stopping;
 
             topVer = exchId.topologyVersion();
+            discoCache = exchFut.discoCache();
 
             updateSeq.setIfGreater(updSeq);
 
@@ -271,7 +277,7 @@ private void beforeExchange0(ClusterNode loc, GridDhtPartitionsExchangeFuture ex
             removeNode(exchId.nodeId());
 
         // In case if node joins, get topology at the time of joining node.
-        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
+        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
         assert oldest != null;
 
@@ -424,7 +430,7 @@ else if (!node2part.nodeId().equals(loc.id())) {
 
             if (!F.isEmpty(nodeIds)) {
                 for (UUID nodeId : nodeIds) {
-                    ClusterNode n = cctx.discovery().node(nodeId);
+                    ClusterNode n = discoCache.node(nodeId);
 
                     if (n != null && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion())) {
                         if (nodes == null)
@@ -450,7 +456,7 @@ else if (!node2part.nodeId().equals(loc.id())) {
      * @return List of nodes for the partition.
      */
     private List<ClusterNode> nodes(int p, AffinityTopologyVersion topVer, GridDhtPartitionState state, GridDhtPartitionState... states) {
-        Collection<UUID> allIds = topVer.topologyVersion() > 0 ? F.nodeIds(CU.allNodes(cctx, topVer)) : null;
+        Collection<UUID> allIds = topVer.topologyVersion() > 0 ? F.nodeIds(discoCache.allNodesWithCaches()) : null;
 
         lock.readLock().lock();
 
@@ -473,7 +479,7 @@ private List<ClusterNode> nodes(int p, AffinityTopologyVersion topVer, GridDhtPa
                     continue;
 
                 if (hasState(p, id, state, states)) {
-                    ClusterNode n = cctx.discovery().node(id);
+                    ClusterNode n = discoCache.node(id);
 
                     if (n != null && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion()))
                         nodes.add(n);
@@ -766,7 +772,7 @@ private void updateLocal(int p, UUID nodeId, GridDhtPartitionState state, long u
         assert nodeId.equals(cctx.localNodeId());
 
         // In case if node joins, get topology at the time of joining node.
-        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
+        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
         // If this node became the oldest node.
         if (oldest.id().equals(cctx.localNodeId())) {
@@ -816,7 +822,7 @@ private void removeNode(UUID nodeId) {
         assert nodeId != null;
         assert lock.writeLock().isHeldByCurrentThread();
 
-        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
+        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
         ClusterNode loc = cctx.localNode();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index ab8e863e90e7d..2c3d7ecc54482 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -72,16 +73,18 @@ public class GridDhtAssignmentFetchFuture extends GridFutureAdapter<GridDhtAffin
      * @param ctx Context.
      * @param cacheName Cache name.
      * @param topVer Topology version.
+     * @param discoCache Discovery cache.
      */
     public GridDhtAssignmentFetchFuture(
         GridCacheSharedContext ctx,
         String cacheName,
-        AffinityTopologyVersion topVer
+        AffinityTopologyVersion topVer,
+        DiscoCache discoCache
     ) {
         this.ctx = ctx;
         this.key = new T2<>(CU.cacheId(cacheName), topVer);
 
-        Collection<ClusterNode> availableNodes = ctx.discovery().cacheAffinityNodes(cacheName, topVer);
+        Collection<ClusterNode> availableNodes = discoCache.cacheAffinityNodes(CU.cacheId(cacheName));
 
         LinkedList<ClusterNode> tmp = new LinkedList<>();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 1b4dcc9293dee..5c3fba08fcf7d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -95,6 +96,9 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
     /** */
     private volatile AffinityTopologyVersion topVer = AffinityTopologyVersion.NONE;
 
+    /** Discovery cache. */
+    private volatile DiscoCache discoCache;
+
     /** */
     private volatile boolean stopping;
 
@@ -151,6 +155,8 @@ public void onReconnected() {
             rebalancedTopVer = AffinityTopologyVersion.NONE;
 
             topVer = AffinityTopologyVersion.NONE;
+
+            discoCache = cctx.discovery().discoCache();
         }
         finally {
             lock.writeLock().unlock();
@@ -293,6 +299,8 @@ private boolean waitForRent() throws IgniteCheckedException {
             rebalancedTopVer = AffinityTopologyVersion.NONE;
 
             topVer = exchId.topologyVersion();
+
+            discoCache = exchFut.discoCache();
         }
         finally {
             lock.writeLock().unlock();
@@ -356,7 +364,7 @@ private boolean waitForRent() throws IgniteCheckedException {
     private void initPartitions0(GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
         ClusterNode loc = cctx.localNode();
 
-        ClusterNode oldest = currentCoordinator();
+        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
         GridDhtPartitionExchangeId exchId = exchFut.exchangeId();
 
@@ -481,7 +489,7 @@ else if (localNode(p, aff))
             if (exchId.isLeft())
                 removeNode(exchId.nodeId());
 
-            ClusterNode oldest = currentCoordinator();
+            ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
             if (log.isDebugEnabled())
                 log.debug("Partition map beforeExchange [exchId=" + exchId + ", fullMap=" + fullMapString() + ']');
@@ -882,7 +890,7 @@ private List<ClusterNode> nodes(int p,
         AffinityTopologyVersion topVer,
         GridDhtPartitionState state,
         GridDhtPartitionState... states) {
-        Collection<UUID> allIds = topVer.topologyVersion() > 0 ? F.nodeIds(CU.affinityNodes(cctx, topVer)) : null;
+        Collection<UUID> allIds = topVer.topologyVersion() > 0 ? F.nodeIds(discoCache.cacheAffinityNodes(cctx.cacheId())) : null;
 
         lock.readLock().lock();
 
@@ -979,7 +987,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
 
     /** {@inheritDoc} */
     @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-    @Nullable @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
+    @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         GridDhtPartitionFullMap partMap,
         @Nullable Map<Integer, Long> cntrMap) {
         if (log.isDebugEnabled())
@@ -1112,7 +1120,7 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
+    @Override public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         GridDhtPartitionMap2 parts,
         @Nullable Map<Integer, Long> cntrMap,
         boolean checkEvictions) {
@@ -1284,7 +1292,8 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
                 List<ClusterNode> affNodes = aff.get(p);
 
                 if (!affNodes.contains(cctx.localNode())) {
-                    Collection<UUID> nodeIds = F.nodeIds(nodes(p, topVer, OWNING));
+                    List<ClusterNode> nodes = nodes(p, topVer, OWNING);
+                    Collection<UUID> nodeIds = F.nodeIds(nodes);
 
                     // If all affinity nodes are owners, then evict partition from local node.
                     if (nodeIds.containsAll(F.nodeIds(affNodes))) {
@@ -1302,15 +1311,13 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
                         int affCnt = affNodes.size();
 
                         if (ownerCnt > affCnt) {
-                            List<ClusterNode> sorted = new ArrayList<>(cctx.discovery().nodes(nodeIds));
-
                             // Sort by node orders in ascending order.
-                            Collections.sort(sorted, CU.nodeComparator(true));
+                            Collections.sort(nodes, CU.nodeComparator(true));
 
-                            int diff = sorted.size() - affCnt;
+                            int diff = nodes.size() - affCnt;
 
                             for (int i = 0; i < diff; i++) {
-                                ClusterNode n = sorted.get(i);
+                                ClusterNode n = nodes.get(i);
 
                                 if (locId.equals(n.id())) {
                                     part.rent(false);
@@ -1335,17 +1342,6 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
         return changed;
     }
 
-    /**
-     * @return Current coordinator node.
-     */
-    @Nullable private ClusterNode currentCoordinator() {
-        ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(topVer);
-
-        assert oldest != null || cctx.kernalContext().clientNode();
-
-        return oldest;
-    }
-
     /**
      * Updates value for single partition.
      *
@@ -1356,7 +1352,7 @@ private boolean checkEvictions(long updateSeq, List<List<ClusterNode>> aff) {
      */
     @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
     private long updateLocal(int p, GridDhtPartitionState state, long updateSeq) {
-        ClusterNode oldest = currentCoordinator();
+        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
         assert oldest != null || cctx.kernalContext().clientNode();
 
@@ -1421,7 +1417,7 @@ private long updateLocal(int p, GridDhtPartitionState state, long updateSeq) {
     private void removeNode(UUID nodeId) {
         assert nodeId != null;
 
-        ClusterNode oldest = CU.oldest(cctx.discovery().serverNodes(topVer));
+        ClusterNode oldest = discoCache.oldestAliveServerNode();
 
         assert oldest != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 44780f18a78eb..c91f881b60378 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -2441,7 +2441,7 @@ private UpdateSingleResult updateSingle(
 
         AffinityTopologyVersion topVer = req.topologyVersion();
 
-        boolean checkReaders = hasNear || ctx.discovery().hasNearCache(name(), topVer);
+        boolean checkReaders = hasNear || ctx.discovery().hasNearCache(ctx.cacheId(), topVer);
 
         boolean readersOnly = false;
 
@@ -2676,7 +2676,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
 
         AffinityTopologyVersion topVer = req.topologyVersion();
 
-        boolean checkReaders = hasNear || ctx.discovery().hasNearCache(name(), topVer);
+        boolean checkReaders = hasNear || ctx.discovery().hasNearCache(ctx.cacheId(), topVer);
 
         CacheStorePartialUpdateException storeErr = null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index e945de958a339..7b5d09b844c31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -44,7 +44,7 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
-import org.apache.ignite.internal.managers.discovery.GridDiscoveryTopologySnapshot;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -103,6 +103,10 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** Dummy reassign flag. */
     private final boolean reassign;
 
+    /** */
+    @GridToStringExclude
+    private volatile DiscoCache discoCache;
+
     /** Discovery event. */
     private volatile DiscoveryEvent discoEvt;
 
@@ -147,9 +151,6 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** */
     private boolean init;
 
-    /** Topology snapshot. */
-    private AtomicReference<GridDiscoveryTopologySnapshot> topSnapshot = new AtomicReference<>();
-
     /** Last committed cache version before next topology version use. */
     private AtomicReference<GridCacheVersion> lastVer = new AtomicReference<>();
 
@@ -331,6 +332,13 @@ public boolean dummyReassign() {
         return (dummy() || forcePreload()) && reassign();
     }
 
+    /**
+     * @return Discovery cache.
+     */
+    public DiscoCache discoCache() {
+        return discoCache;
+    }
+
     /**
      * @param cacheId Cache ID to check.
      * @param topVer Topology version.
@@ -374,11 +382,13 @@ public boolean onAdded() {
      *
      * @param exchId Exchange ID.
      * @param discoEvt Discovery event.
+     * @param discoCache Discovery data cache.
      */
-    public void onEvent(GridDhtPartitionExchangeId exchId, DiscoveryEvent discoEvt) {
+    public void onEvent(GridDhtPartitionExchangeId exchId, DiscoveryEvent discoEvt, DiscoCache discoCache) {
         assert exchId.equals(this.exchId);
 
         this.discoEvt = discoEvt;
+        this.discoCache = discoCache;
 
         evtLatch.countDown();
     }
@@ -435,7 +445,9 @@ public void init() throws IgniteInterruptedCheckedException {
         assert !dummy && !forcePreload : this;
 
         try {
-            srvNodes = new ArrayList<>(cctx.discovery().serverNodes(topologyVersion()));
+            discoCache.updateAlives(cctx.discovery());
+
+            srvNodes = new ArrayList<>(discoCache.serverNodes());
 
             remaining.addAll(F.nodeIds(F.view(srvNodes, F.remoteNodes(cctx.localNodeId()))));
 
@@ -550,7 +562,7 @@ private void updateTopologies(boolean crd) throws IgniteCheckedException {
                     exchId.topologyVersion().equals(cacheCtx.startTopologyVersion());
 
                 if (updateTop && clientTop != null)
-                    cacheCtx.topology().update(exchId, clientTop.partitionMap(true), clientTop.updateCounters(false));
+                    top.update(exchId, clientTop.partitionMap(true), clientTop.updateCounters(false));
             }
 
             top.updateTopologyVersion(exchId, this, updSeq, stopping(cacheCtx.cacheId()));
@@ -842,7 +854,7 @@ private void warnNoAffinityNodes() {
         List<String> cachesWithoutNodes = null;
 
         for (String name : cctx.cache().cacheNames()) {
-            if (cctx.discovery().cacheAffinityNodes(name, topologyVersion()).isEmpty()) {
+            if (discoCache.cacheAffinityNodes(name).isEmpty()) {
                 if (cachesWithoutNodes == null)
                     cachesWithoutNodes = new ArrayList<>();
 
@@ -1096,7 +1108,6 @@ private void sendPartitions(ClusterNode oldestNode) {
      * Cleans up resources to avoid excessive memory usage.
      */
     public void cleanUp() {
-        topSnapshot.set(null);
         singleMsgs.clear();
         fullMsgs.clear();
         crd = null;
@@ -1252,7 +1263,7 @@ private void onAllReceived(boolean discoThread) {
         try {
             assert crd.isLocal();
 
-            if (!crd.equals(cctx.discovery().serverNodes(topologyVersion()).get(0))) {
+            if (!crd.equals(discoCache.serverNodes().get(0))) {
                 for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
                     if (!cacheCtx.isLocal())
                         cacheCtx.topology().beforeExchange(GridDhtPartitionsExchangeFuture.this, !centralizedAff);
@@ -1559,6 +1570,8 @@ public void onNodeLeft(final ClusterNode node) {
 
                         ClusterNode crd0;
 
+                        discoCache.updateAlives(node);
+
                         synchronized (mux) {
                             if (!srvNodes.remove(node))
                                 return;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index d26242db68f0d..99146aab1d88e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -48,7 +48,6 @@
 import org.apache.ignite.configuration.DeploymentMode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
-import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.GridClosureCallMode;
 import org.apache.ignite.internal.GridKernalContext;
@@ -58,8 +57,9 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
-import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -167,7 +167,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
     private IgniteInternalCache<Object, Object> cache;
 
     /** Topology listener. */
-    private GridLocalEventListener topLsnr = new TopologyListener();
+    private DiscoveryEventListener topLsnr = new TopologyListener();
 
     static {
         Set<IgniteProductVersion> versions = new TreeSet<>(new Comparator<IgniteProductVersion>() {
@@ -251,7 +251,7 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         cache = ctx.cache().utilityCache();
 
         if (!ctx.clientNode())
-            ctx.event().addLocalEventListener(topLsnr, EVTS);
+            ctx.event().addDiscoveryEventListener(topLsnr, EVTS);
 
         try {
             if (ctx.deploy().enabled())
@@ -314,7 +314,7 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         busyLock.block();
 
         if (!ctx.clientNode())
-            ctx.event().removeLocalEventListener(topLsnr);
+            ctx.event().removeDiscoveryEventListener(topLsnr);
 
         Collection<ServiceContextImpl> ctxs = new ArrayList<>();
 
@@ -1568,9 +1568,9 @@ private void onDeployment(final GridServiceDeployment dep, final AffinityTopolog
     /**
      * Topology listener.
      */
-    private class TopologyListener implements GridLocalEventListener {
+    private class TopologyListener implements DiscoveryEventListener {
         /** {@inheritDoc} */
-        @Override public void onEvent(Event evt) {
+        @Override public void onEvent(DiscoveryEvent evt, final DiscoCache discoCache) {
             if (!busyLock.enterBusy())
                 return;
 
@@ -1588,11 +1588,14 @@ private class TopologyListener implements GridLocalEventListener {
                     }
                 }
                 else
-                    topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
+                    topVer = new AffinityTopologyVersion((evt).topologyVersion(), 0);
 
                 depExe.execute(new BusyRunnable() {
                     @Override public void run0() {
-                        ClusterNode oldest = ctx.discovery().oldestAliveCacheServerNode(topVer);
+                        // In case the cache instance isn't tracked by DiscoveryManager anymore.
+                        discoCache.updateAlives(ctx.discovery());
+
+                        ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache();
 
                         if (oldest != null && oldest.isLocal()) {
                             final Collection<GridServiceDeployment> retries = new ConcurrentLinkedQueue<>();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
index 31b4bc7ca6682..f0c50ebeb3bf1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
@@ -23,16 +23,11 @@
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.GridCacheUtils;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -90,8 +85,8 @@ public class GridDiscoveryManagerAliveCacheSelfTest extends GridCommonAbstractTe
     }
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
         CacheConfiguration cCfg = defaultCacheConfiguration();
 
@@ -103,7 +98,7 @@ public class GridDiscoveryManagerAliveCacheSelfTest extends GridCommonAbstractTe
 
         TcpDiscoverySpi disc = new TcpDiscoverySpi();
 
-        if (clientMode && ((gridName.charAt(gridName.length() - 1) - '0') & 1) != 0)
+        if (clientMode && ((igniteInstanceName.charAt(igniteInstanceName.length() - 1) - '0') & 1) != 0)
             cfg.setClientMode(true);
         else
             disc.setMaxMissedClientHeartbeats(50);
@@ -158,8 +153,6 @@ private void doTestAlive() throws Exception {
             stopTempNodes();
 
             latch.await();
-
-            validateAlives();
         }
     }
 
@@ -199,55 +192,6 @@ private void awaitDiscovery(long nodesCnt) throws InterruptedException {
         }
     }
 
-    /**
-     * Validates that all node collections contain actual information.
-     */
-    @SuppressWarnings("SuspiciousMethodCalls")
-    private void validateAlives() {
-        for (Ignite g : alive) {
-            log.info("Validate node: " + g.name());
-
-            assertEquals("Unexpected nodes number for node: " + g.name(), PERM_NODES_CNT, g.cluster().nodes().size());
-        }
-
-        for (final Ignite g : alive) {
-            IgniteKernal k = (IgniteKernal)g;
-
-            GridDiscoveryManager discoMgr = k.context().discovery();
-
-            final Collection<ClusterNode> currTop = g.cluster().nodes();
-
-            long currVer = discoMgr.topologyVersion();
-
-            long startVer = discoMgr.localNode().order();
-
-            for (long v = currVer; v > currVer - GridDiscoveryManager.DISCOVERY_HISTORY_SIZE && v >= startVer; v--) {
-                F.forAll(discoMgr.aliveCacheNodes(null, new AffinityTopologyVersion(v)),
-                    new IgnitePredicate<ClusterNode>() {
-                        @Override public boolean apply(ClusterNode e) {
-                            return currTop.contains(e);
-                        }
-                    });
-
-                F.forAll(discoMgr.aliveRemoteCacheNodes(null, new AffinityTopologyVersion(v)),
-                    new IgnitePredicate<ClusterNode>() {
-                        @Override public boolean apply(ClusterNode e) {
-                            return currTop.contains(e) || g.cluster().localNode().equals(e);
-                        }
-                    });
-
-                GridCacheSharedContext<?, ?> ctx = k.context().cache().context();
-
-                ClusterNode oldest =
-                    ctx.discovery().oldestAliveCacheServerNode(new AffinityTopologyVersion(currVer));
-
-                assertNotNull(oldest);
-
-                assertTrue(currTop.contains(oldest));
-            }
-        }
-    }
-
     /**
      * Starts temporary nodes.
      *
@@ -293,4 +237,4 @@ private void stopTempNodes() {
             G.stop(g.name(), false);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
index ba8fa5b6e15a3..5de29106f0ee3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
@@ -54,10 +54,10 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA
     private static boolean binaryMarshallerEnabled;
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
-        if (gridName.equals(getTestGridName(1)))
+        if (igniteInstanceName.equals(getTestGridName(1)))
             cfg.setClientMode(true);
 
         if (binaryMarshallerEnabled)
@@ -160,7 +160,7 @@ private void doTestUseDefaultSuid(String first, String second, boolean fail) thr
                 if (fail)
                     fail("Node should not join");
             }
-            catch (Exception e) {
+            catch (Exception ignored) {
                 if (!fail)
                     fail("Node should join");
             }
@@ -215,7 +215,7 @@ private void doTestUseStrSerVer2(String first, String second, boolean fail) thro
                 if (fail)
                     fail("Node should not join");
             }
-            catch (Exception e) {
+            catch (Exception ignored) {
                 if (!fail)
                     fail("Node should join");
             }
@@ -346,8 +346,8 @@ private void testPreferIpV4Stack(boolean preferIpV4) throws Exception {
      */
     public static class RegularDiscovery extends GridDiscoveryManagerAttributesSelfTest {
         /** {@inheritDoc} */
-        @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-            IgniteConfiguration cfg = super.getConfiguration(gridName);
+        @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+            IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
             ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setForceServerMode(true);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerSelfTest.java
deleted file mode 100644
index c9179d44c0060..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerSelfTest.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.managers.discovery;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.NearCacheConfiguration;
-import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
-
-/**
- *
- */
-public abstract class GridDiscoveryManagerSelfTest extends GridCommonAbstractTest {
-    /** */
-    private static final String CACHE_NAME = "cache";
-
-    /** */
-    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        stopAllGrids();
-    }
-
-    /** {@inheritDoc} */
-    @SuppressWarnings("IfMayBeConditional")
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        CacheConfiguration ccfg1 = defaultCacheConfiguration();
-
-        ccfg1.setName(CACHE_NAME);
-
-        CacheConfiguration ccfg2 = defaultCacheConfiguration();
-
-        ccfg2.setName(null);
-
-        if (gridName.equals(getTestGridName(1)))
-            cfg.setClientMode(true);
-        else {
-            ccfg1.setNearConfiguration(null);
-            ccfg2.setNearConfiguration(null);
-
-            ccfg1.setCacheMode(PARTITIONED);
-            ccfg2.setCacheMode(PARTITIONED);
-
-            cfg.setCacheConfiguration(ccfg1, ccfg2);
-        }
-
-        TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi();
-
-        discoverySpi.setIpFinder(IP_FINDER);
-
-        cfg.setDiscoverySpi(discoverySpi);
-
-        return cfg;
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testHasNearCache() throws Exception {
-        IgniteKernal g0 = (IgniteKernal)startGrid(0); // PARTITIONED_ONLY cache.
-
-        AffinityTopologyVersion none = new AffinityTopologyVersion(-1);
-        AffinityTopologyVersion one = new AffinityTopologyVersion(1);
-        AffinityTopologyVersion two = new AffinityTopologyVersion(2, 2);
-        AffinityTopologyVersion three = new AffinityTopologyVersion(3);
-        AffinityTopologyVersion four = new AffinityTopologyVersion(4);
-        AffinityTopologyVersion five = new AffinityTopologyVersion(5);
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, none));
-        assertFalse(g0.context().discovery().hasNearCache(null, none));
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, one));
-        assertFalse(g0.context().discovery().hasNearCache(null, one));
-
-        IgniteKernal g1 = (IgniteKernal)startGrid(1); // NEAR_ONLY cache.
-
-        grid(1).createNearCache(null, new NearCacheConfiguration());
-
-        grid(1).createNearCache(CACHE_NAME, new NearCacheConfiguration());
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, one));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertFalse(g0.context().discovery().hasNearCache(null, one));
-        assertTrue(g0.context().discovery().hasNearCache(null, two));
-
-        assertTrue(g1.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertTrue(g1.context().discovery().hasNearCache(null, two));
-
-        IgniteKernal g2 = (IgniteKernal)startGrid(2); // PARTITIONED_ONLY cache.
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, one));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertFalse(g0.context().discovery().hasNearCache(null, one));
-        assertTrue(g0.context().discovery().hasNearCache(null, two));
-        assertTrue(g0.context().discovery().hasNearCache(null, three));
-
-        assertTrue(g1.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertTrue(g1.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertTrue(g1.context().discovery().hasNearCache(null, two));
-        assertTrue(g1.context().discovery().hasNearCache(null, three));
-
-        assertTrue(g2.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertTrue(g2.context().discovery().hasNearCache(null, three));
-
-        stopGrid(2);
-
-        // Wait all nodes are on version 4.
-        for (;;) {
-            if (F.forAll(
-                Ignition.allGrids(),
-                new IgnitePredicate<Ignite>() {
-                    @Override public boolean apply(Ignite ignite) {
-                        return ignite.cluster().topologyVersion() == 4;
-                    }
-                }))
-                break;
-
-            Thread.sleep(1000);
-        }
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, one));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, four));
-        assertFalse(g0.context().discovery().hasNearCache(null, one));
-        assertTrue(g0.context().discovery().hasNearCache(null, two));
-        assertTrue(g0.context().discovery().hasNearCache(null, three));
-        assertTrue(g0.context().discovery().hasNearCache(null, four));
-
-        assertTrue(g1.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertTrue(g1.context().discovery().hasNearCache(CACHE_NAME, four));
-        assertTrue(g1.context().discovery().hasNearCache(null, three));
-        assertTrue(g1.context().discovery().hasNearCache(null, four));
-
-        stopGrid(1);
-
-        // Wait all nodes are on version 5.
-        for (;;) {
-            if (F.forAll(
-                Ignition.allGrids(),
-                new IgnitePredicate<Ignite>() {
-                    @Override public boolean apply(Ignite ignite) {
-                        return ignite.cluster().topologyVersion() == 5;
-                    }
-                }))
-                break;
-
-            Thread.sleep(1000);
-        }
-
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, one));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, two));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, three));
-        assertTrue(g0.context().discovery().hasNearCache(CACHE_NAME, four));
-        assertFalse(g0.context().discovery().hasNearCache(CACHE_NAME, five));
-
-        assertFalse(g0.context().discovery().hasNearCache(null, one));
-        assertTrue(g0.context().discovery().hasNearCache(null, two));
-        assertTrue(g0.context().discovery().hasNearCache(null, three));
-        assertTrue(g0.context().discovery().hasNearCache(null, four));
-        assertFalse(g0.context().discovery().hasNearCache(null, five));
-    }
-
-    /**
-     *
-     */
-    public static class RegularDiscovery extends GridDiscoveryManagerSelfTest {
-        /** {@inheritDoc} */
-        @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-            IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-            ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setForceServerMode(true);
-
-            return cfg;
-        }
-    }
-
-    /**
-     *
-     */
-    public static class ClientDiscovery extends GridDiscoveryManagerSelfTest {
-        // No-op.
-    }
-}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/IgniteTopologyPrintFormatSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/IgniteTopologyPrintFormatSelfTest.java
index 58992afbbd1a7..86ad4589eaa85 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/IgniteTopologyPrintFormatSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/IgniteTopologyPrintFormatSelfTest.java
@@ -50,16 +50,16 @@ public class IgniteTopologyPrintFormatSelfTest extends GridCommonAbstractTest {
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
         TcpDiscoverySpi disc = new TcpDiscoverySpi();
         disc.setIpFinder(IP_FINDER);
 
-        if (gridName.endsWith("client"))
+        if (igniteInstanceName.endsWith("client"))
             cfg.setClientMode(true);
 
-        if (gridName.endsWith("client_force_server")) {
+        if (igniteInstanceName.endsWith("client_force_server")) {
             cfg.setClientMode(true);
             disc.setForceServerMode(true);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index b28619cd91ed5..985dddb93bf4d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -42,7 +42,6 @@
 import org.apache.ignite.internal.managers.deployment.GridDeploymentManagerStopSelfTest;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerAliveCacheSelfTest;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerAttributesSelfTest;
-import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerSelfTest;
 import org.apache.ignite.internal.managers.discovery.IgniteTopologyPrintFormatSelfTest;
 import org.apache.ignite.internal.managers.events.GridEventStorageManagerSelfTest;
 import org.apache.ignite.internal.managers.swapspace.GridSwapSpaceManagerSelfTest;
@@ -111,8 +110,6 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridDiscoveryManagerAttributesSelfTest.RegularDiscovery.class);
         suite.addTestSuite(GridDiscoveryManagerAttributesSelfTest.ClientDiscovery.class);
         suite.addTestSuite(GridDiscoveryManagerAliveCacheSelfTest.class);
-        suite.addTestSuite(GridDiscoveryManagerSelfTest.RegularDiscovery.class);
-        suite.addTestSuite(GridDiscoveryManagerSelfTest.ClientDiscovery.class);
         suite.addTestSuite(GridDiscoveryEventSelfTest.class);
         suite.addTestSuite(GridPortProcessorSelfTest.class);
         suite.addTestSuite(GridHomePathSelfTest.class);

From 6775f646df127f5cdbabf7a154b65856f38afa1e Mon Sep 17 00:00:00 2001
From: Dmitriy Shabalin <dshabalin@gridgain.com>
Date: Wed, 22 Mar 2017 10:37:05 +0700
Subject: [PATCH 071/516] IGNITE-4686 Added ability to group registered users
 in admin panel.

(cherry picked from commit 827befb)
---
 modules/web-console/frontend/app/app.js       |   1 +
 .../form-field-datepicker.pug                 |  12 +-
 .../list-of-registered-users/index.js         |   2 +
 .../list-of-registered-users.column-defs.js   |  48 ++---
 .../list-of-registered-users.controller.js    | 193 +++++++++++++-----
 .../list-of-registered-users.scss             |  28 +++
 .../list-of-registered-users.tpl.pug          |  50 +++--
 .../ui-grid-header/ui-grid-header.scss        |   6 +
 .../ui-grid-header/ui-grid-header.tpl.pug     |   4 +-
 .../ui-grid-settings/ui-grid-settings.scss    |  10 +
 .../frontend/app/primitives/badge/index.scss  |  36 ++++
 .../frontend/app/primitives/index.js          |  19 ++
 .../frontend/app/primitives/tabs/index.scss   |  73 +++++++
 .../frontend/public/stylesheets/style.scss    |   2 +
 14 files changed, 389 insertions(+), 95 deletions(-)
 create mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.scss
 create mode 100644 modules/web-console/frontend/app/primitives/badge/index.scss
 create mode 100644 modules/web-console/frontend/app/primitives/index.js
 create mode 100644 modules/web-console/frontend/app/primitives/tabs/index.scss

diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 1e21d247c060a..26d3ad586c2e7 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -16,6 +16,7 @@
  */
 
 import '../public/stylesheets/style.scss';
+import '../app/primitives';
 import './components/ui-grid-header/ui-grid-header.scss';
 import './components/ui-grid-settings/ui-grid-settings.scss';
 import './components/form-field-datepicker/form-field-datepicker.scss';
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
index c9d382cb36cc4..d70476f373709 100644
--- a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
+++ b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin ignite-form-field-datepicker(label, model, name, disabled, required, placeholder, tip)
+mixin ignite-form-field-datepicker(label, model, name, mindate, maxdate, disabled, required, placeholder, tip)
     mixin form-field-input()
         input.form-control(
             id=`{{ ${name} }}Input`
@@ -30,8 +30,10 @@ mixin ignite-form-field-datepicker(label, model, name, disabled, required, place
             bs-datepicker
             data-date-format='MMM yyyy'
             data-start-view='1'
-            data-min-view='1' 
-            data-max-date='today'
+            data-min-view='1'
+
+            data-min-date=mindate ? `{{ ${mindate} }}` : false
+            data-max-date=maxdate ? `{{ ${maxdate} }}` : `today`
 
             data-container='body > .wrapper'
 
@@ -43,7 +45,9 @@ mixin ignite-form-field-datepicker(label, model, name, disabled, required, place
         )&attributes(attributes.attributes)
 
     .ignite-form-field
-        +ignite-form-field__label(label, name, required)
+        if name
+            +ignite-form-field__label(label, name, required)
+
         .ignite-form-field__control
             if tip
                 i.tipField.icon-help(bs-tooltip='' data-title=tip)
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/index.js b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
index 22a89da3b6a24..4e5061f352e0a 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/index.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/index.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import './list-of-registered-users.scss';
+
 import templateUrl from './list-of-registered-users.tpl.pug';
 import controller from './list-of-registered-users.controller';
 
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
index e6ba842aa055b..e859acf7d0ec6 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
@@ -49,32 +49,32 @@ const ACTIONS_TEMPLATE = `
 const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
 
 export default [
-    {displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 65, width: 65, enableFiltering: false, enableSorting: false, pinnedLeft: true},
-    {displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by name...' }, pinnedLeft: true},
-    {displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
-    {displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 160, enableFiltering: true},
-    {displayName: 'Country', categoryDisplayName: 'Country', field: 'countryCode', minWidth: 80, enableFiltering: true},
-    {displayName: 'Last login', categoryDisplayName: 'Last login', field: 'lastLogin', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 105, width: 105, enableFiltering: false, visible: false},
-    {displayName: 'Last activity', categoryDisplayName: 'Last activity', field: 'lastActivity', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 105, width: 105, enableFiltering: false, visible: true, sort: { direction: 'desc', priority: 0 }},
+    {name: 'actions', displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false},
+    {name: 'user', displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by name...' }},
+    {name: 'email', displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
+    {name: 'company', displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by company...' }},
+    {name: 'country', displayName: 'Country', categoryDisplayName: 'Country', field: 'countryCode', minWidth: 80, enableFiltering: true, filter: { placeholder: 'Filter by country...' }},
+    {name: 'lastlogin', displayName: 'Last login', categoryDisplayName: 'Last login', field: 'lastLogin', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 105, width: 105, enableFiltering: false, visible: false},
+    {name: 'lastactivity', displayName: 'Last activity', categoryDisplayName: 'Last activity', field: 'lastActivity', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 115, width: 115, enableFiltering: false, visible: true, sort: { direction: 'desc', priority: 0 }},
     // Configurations
-    {displayName: 'Clusters count', categoryDisplayName: 'Configurations', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: 'counters.clusters', type: 'number', headerTooltip: 'Clusters count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
-    {displayName: 'Models count', categoryDisplayName: 'Configurations', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: 'counters.models', type: 'number', headerTooltip: 'Models count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
-    {displayName: 'Caches count', categoryDisplayName: 'Configurations', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: 'counters.caches', type: 'number', headerTooltip: 'Caches count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
-    {displayName: 'IGFS count', categoryDisplayName: 'Configurations', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: 'counters.igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 50, width: 50, enableFiltering: false, visible: false},
+    {name: 'cfg_clusters', displayName: 'Clusters count', categoryDisplayName: 'Configurations', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: 'counters.clusters', type: 'number', headerTooltip: 'Clusters count', minWidth: 55, width: 55, enableFiltering: false, visible: false},
+    {name: 'cfg_models', displayName: 'Models count', categoryDisplayName: 'Configurations', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: 'counters.models', type: 'number', headerTooltip: 'Models count', minWidth: 55, width: 55, enableFiltering: false, visible: false},
+    {name: 'cfg_caches', displayName: 'Caches count', categoryDisplayName: 'Configurations', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: 'counters.caches', type: 'number', headerTooltip: 'Caches count', minWidth: 55, width: 55, enableFiltering: false, visible: false},
+    {name: 'cfg_igfs', displayName: 'IGFS count', categoryDisplayName: 'Configurations', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: 'counters.igfs', type: 'number', headerTooltip: 'IGFS count', minWidth: 55, width: 55, enableFiltering: false, visible: false},
     // Activities Total
-    {displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Total count of configuration usages', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Total count of queries usages', minWidth: 50, width: 50, enableFiltering: false},
-    {displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Total count of demo startup', minWidth: 60, width: 60, enableFiltering: false},
-    {displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Total count of agent downloads', minWidth: 55, width: 55, enableFiltering: false},
-    {displayName: 'Starts', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Total count of agent startup', minWidth: 60, width: 60, enableFiltering: false},
+    {name: 'cfg', displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', headerTooltip: 'Total count of configuration usages', minWidth: 55, width: 55, enableFiltering: false},
+    {name: 'qry', displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', headerTooltip: 'Total count of queries usages', minWidth: 55, width: 55, enableFiltering: false},
+    {name: 'demo', displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', headerTooltip: 'Total count of demo startup', minWidth: 65, width: 65, enableFiltering: false},
+    {name: 'dnld', displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', headerTooltip: 'Total count of agent downloads', minWidth: 55, width: 55, enableFiltering: false},
+    {name: 'starts', displayName: 'Starts', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', headerTooltip: 'Total count of agent startup', minWidth: 65, width: 65, enableFiltering: false},
     // Activities Configuration
-    {displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration clusters', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration model', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', headerTooltip: 'Configuration caches', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'IGFS', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/igfs"] || 0', type: 'number', headerTooltip: 'Configuration IGFS', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', headerTooltip: 'Configuration summary', minWidth: 50, width: 80, enableFiltering: false, visible: false},
+    {name: 'clusters', displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', headerTooltip: 'Configuration clusters', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'model', displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', headerTooltip: 'Configuration model', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'caches', displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', headerTooltip: 'Configuration caches', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'igfs', displayName: 'IGFS', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/igfs"] || 0', type: 'number', headerTooltip: 'Configuration IGFS', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'summary', displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', headerTooltip: 'Configuration summary', minWidth: 55, width: 80, enableFiltering: false, visible: false},
     // Activities Queries
-    {displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', headerTooltip: 'Query executions', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', headerTooltip: 'Query explain executions', minWidth: 50, width: 80, enableFiltering: false, visible: false},
-    {displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', headerTooltip: 'Scan query executions', minWidth: 50, width: 80, enableFiltering: false, visible: false}
+    {name: 'execute', displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', headerTooltip: 'Query executions', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'explain', displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', headerTooltip: 'Query explain executions', minWidth: 55, width: 80, enableFiltering: false, visible: false},
+    {name: 'scan', displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', headerTooltip: 'Scan query executions', minWidth: 55, width: 80, enableFiltering: false, visible: false}
 ];
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
index 54971b142b014..acf76faf44918 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.controller.js
@@ -30,34 +30,21 @@ const rowTemplate = `<div
   ui-grid-cell/>`;
 
 export default class IgniteListOfRegisteredUsersCtrl {
-    static $inject = ['$scope', '$state', '$filter', 'User', 'uiGridConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
+    static $inject = ['$scope', '$state', '$filter', 'User', 'uiGridGroupingConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
 
-    constructor($scope, $state, $filter, User, uiGridConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
+    constructor($scope, $state, $filter, User, uiGridGroupingConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
         const $ctrl = this;
 
-        const companySelectOptions = [];
-        const countrySelectOptions = [];
-
         const dtFilter = $filter('date');
 
+        $ctrl.groupBy = 'user';
+
         $ctrl.params = {
-            startDate: new Date()
+            startDate: new Date(),
+            endDate: new Date()
         };
 
-        const columnCompany = _.find(columnDefs, { displayName: 'Company' });
-        const columnCountry = _.find(columnDefs, { displayName: 'Country' });
-
-        columnCompany.filter = {
-            selectOptions: companySelectOptions,
-            type: uiGridConstants.filter.SELECT,
-            condition: uiGridConstants.filter.EXACT
-        };
-
-        columnCountry.filter = {
-            selectOptions: countrySelectOptions,
-            type: uiGridConstants.filter.SELECT,
-            condition: uiGridConstants.filter.EXACT
-        };
+        $ctrl.uiGridGroupingConstants = uiGridGroupingConstants;
 
         const becomeUser = (user) => {
             AdminData.becomeUser(user._id)
@@ -105,6 +92,11 @@ export default class IgniteListOfRegisteredUsersCtrl {
             return renderableRows;
         };
 
+        $ctrl._userGridOptions = {
+            columnDefs,
+            categories
+        };
+
         $ctrl.gridOptions = {
             data: [],
             columnVirtualizationThreshold: 30,
@@ -134,19 +126,6 @@ export default class IgniteListOfRegisteredUsersCtrl {
             }
         };
 
-        const usersToFilterOptions = (column) => {
-            return _.sortBy(
-                _.map(
-                    _.groupBy($ctrl.gridOptions.data, (usr) => {
-                        const fld = usr[column];
-
-                        return _.isNil(fld) ? fld : fld.toUpperCase();
-                    }),
-                    (arr, value) => ({label: `${_.head(arr)[column] || 'Not set'} (${arr.length})`, value})
-                ),
-                'value');
-        };
-
         /**
          * @param {{startDate: number, endDate: number}} params
          */
@@ -154,31 +133,32 @@ export default class IgniteListOfRegisteredUsersCtrl {
             AdminData.loadUsers(params)
                 .then((data) => $ctrl.gridOptions.data = data)
                 .then((data) => {
-                    companySelectOptions.length = 0;
-                    countrySelectOptions.length = 0;
-
-                    companySelectOptions.push(...usersToFilterOptions('company'));
-                    countrySelectOptions.push(...usersToFilterOptions('countryCode'));
-
                     this.gridApi.grid.refresh();
 
+                    this.companies = _.values(_.groupBy(data, (b) => b.company.toLowerCase()));
+                    this.countries = _.values(_.groupBy(data, (b) => b.countryCode));
+
                     return data;
-                })
-                .then((data) => $ctrl.adjustHeight(data.length));
+                });
+        };
+
+        const fitlerDates = (sdt, edt) => {
+            $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(sdt, 'yyyy_MM')}.csv`;
+
+            const startDate = Date.UTC(sdt.getFullYear(), sdt.getMonth(), 1);
+            const endDate = Date.UTC(edt.getFullYear(), edt.getMonth() + 1, 1);
+
+            reloadUsers({ startDate, endDate });
         };
 
         $scope.$watch(() => $ctrl.params.companiesExclude, () => {
             $ctrl.gridApi.grid.refreshRows();
         });
 
-        $scope.$watch(() => $ctrl.params.startDate, (dt) => {
-            $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(dt, 'yyyy_MM')}.csv`;
+        $scope.$watch(() => $ctrl.params.startDate, (sdt) => fitlerDates(sdt, $ctrl.params.endDate));
+        $scope.$watch(() => $ctrl.params.endDate, (edt) => fitlerDates($ctrl.params.startDate, edt));
 
-            const startDate = Date.UTC(dt.getFullYear(), dt.getMonth(), 1);
-            const endDate = Date.UTC(dt.getFullYear(), dt.getMonth() + 1, 1);
-
-            reloadUsers({ startDate, endDate });
-        });
+        $scope.$watch(() => $ctrl.gridApi.grid.getVisibleRows().length, (length) => $ctrl.adjustHeight(length >= 20 ? 20 : length));
     }
 
     adjustHeight(rows) {
@@ -235,4 +215,121 @@ export default class IgniteListOfRegisteredUsersCtrl {
     exportCsv() {
         this.gridApi.exporter.csvExport('visible', 'visible');
     }
+
+    groupByUser() {
+        this.groupBy = 'user';
+
+        this.gridApi.grouping.clearGrouping();
+
+        this.gridOptions.categories = this._userGridOptions.categories;
+        this.gridOptions.columnDefs = this._userGridOptions.columnDefs;
+    }
+
+    groupByCompany() {
+        this.groupBy = 'company';
+
+        this.gridApi.grouping.clearGrouping();
+        this.gridApi.grouping.groupColumn('company');
+        this.gridApi.grouping.aggregateColumn('user', this.uiGridGroupingConstants.aggregation.COUNT);
+
+        if (this._companyGridOptions) {
+            this.gridOptions.categories = this._companyGridOptions.categories;
+            this.gridOptions.columnDefs = this._companyGridOptions.columnDefs;
+
+            return;
+        }
+
+        const _categories = _.cloneDeep(categories);
+        const _columnDefs = _.cloneDeep(columnDefs);
+
+        // Cut company category;
+        const company = _categories.splice(3, 1)[0];
+
+        // Hide Actions category;
+        _categories.splice(0, 1);
+
+        _.forEach(_.filter(_columnDefs, {displayName: 'Actions'}), (col) => {
+            col.visible = false;
+        });
+
+        // Add company as first column;
+        _categories.unshift(company);
+
+        _.forEach(_columnDefs, (col) => {
+            col.enableSorting = true;
+
+            if (col.type !== 'number')
+                return;
+
+            col.treeAggregationType = this.uiGridGroupingConstants.aggregation.SUM;
+            col.customTreeAggregationFinalizerFn = (agg) => agg.rendered = agg.value;
+        });
+
+        // Set grouping to last activity column
+        const lastactivity = _.find(_columnDefs, { name: 'lastactivity' });
+
+        if (_.nonNil(lastactivity)) {
+            lastactivity.treeAggregationType = this.uiGridGroupingConstants.aggregation.MAX;
+            lastactivity.customTreeAggregationFinalizerFn = (agg) => agg.rendered = agg.value;
+        }
+
+        this._companyGridOptions = {
+            categories: this.gridOptions.categories = _categories,
+            columnDefs: this.gridOptions.columnDefs = _columnDefs
+        };
+    }
+
+    groupByCountry() {
+        this.groupBy = 'country';
+
+        this.gridApi.grouping.clearGrouping();
+        this.gridApi.grouping.groupColumn('country');
+        this.gridApi.grouping.aggregateColumn('user', this.uiGridGroupingConstants.aggregation.COUNT);
+
+        if (this._countryGridOptions) {
+            this.gridOptions.categories = this._countryGridOptions.categories;
+            this.gridOptions.columnDefs = this._countryGridOptions.columnDefs;
+
+            return;
+        }
+
+        const _categories = _.cloneDeep(categories);
+        const _columnDefs = _.cloneDeep(columnDefs);
+
+        // Cut country category;
+        const country = _categories.splice(4, 1)[0];
+
+        // Hide Actions category;
+        _categories.splice(0, 1);
+
+        _.forEach(_.filter(_columnDefs, {displayName: 'Actions'}), (col) => {
+            col.visible = false;
+        });
+
+        // Add company as first column;
+        _categories.unshift(country);
+
+        _.forEach(_columnDefs, (col) => {
+            col.enableSorting = true;
+
+            if (col.type !== 'number')
+                return;
+
+            col.treeAggregationType = this.uiGridGroupingConstants.aggregation.SUM;
+            col.customTreeAggregationFinalizerFn = (agg) => agg.rendered = agg.value;
+        });
+
+        // Set grouping to last activity column
+        const lastactivity = _.find(_columnDefs, { name: 'lastactivity' });
+
+        if (_.nonNil(lastactivity)) {
+            lastactivity.treeAggregationType = this.uiGridGroupingConstants.aggregation.MAX;
+            lastactivity.customTreeAggregationFinalizerFn = (agg) => agg.rendered = agg.value;
+        }
+
+        this._countryGridOptions = {
+            categories: this.gridOptions.categories = _categories,
+            columnDefs: this.gridOptions.columnDefs = _columnDefs
+        };
+    }
 }
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.scss b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.scss
new file mode 100644
index 0000000000000..8059d7084102c
--- /dev/null
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.scss
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.list-of-registered-users {
+  & > a {
+    display: inline-block;
+    margin: 10px;
+    margin-left: 0;
+
+    &.active {
+      font-weight: bold;
+    }
+  }
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
index 52975b98dc0ee..ec4b4fda0543d 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
@@ -34,25 +34,39 @@ mixin grid-settings()
         li
             a(ng-click='$hide()') Close
 
-.panel.panel-default
-    .panel-heading.ui-grid-settings
-        +grid-settings
-        label Total users: 
-            strong {{ $ctrl.gridOptions.data.length }}&nbsp;&nbsp;&nbsp;
-        label Showing users:
-            strong {{ $ctrl.gridApi.grid.getVisibleRows().length }}
-            sub(ng-show='users.length === $ctrl.gridApi.grid.getVisibleRows().length') all
-
-        form.pull-right(ng-form=form novalidate)
-            -var form = 'admin'
 
-            button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
+.list-of-registered-users
+    ul.tabs
+        li(role='presentation' ng-class='{ active: $ctrl.groupBy === "user" }') 
+            a(ng-click='$ctrl.groupByUser()') 
+                span Users
+                span.badge {{ $ctrl.gridOptions.data.length }}
+        li(role='presentation' ng-class='{ active: $ctrl.groupBy === "company" }')
+            a(ng-click='$ctrl.groupByCompany()') 
+                span Companies
+                span.badge {{ $ctrl.companies.length }}
+        li(role='presentation' ng-class='{ active: $ctrl.groupBy === "country" }')
+            a(ng-click='$ctrl.groupByCountry()')
+                span Countries
+                span.badge {{ $ctrl.countries.length }}
+
+    .panel.panel-default
+        .panel-heading.ui-grid-settings
+            +grid-settings
+            label(ng-show='$ctrl.groupBy === "user"') Showing users:&nbsp;
+                strong {{ $ctrl.gridApi.grid.getVisibleRows().length }}
+                sub(ng-show='users.length === $ctrl.gridApi.grid.getVisibleRows().length') all
+
+            -var form = 'admin'
+            form.pull-right(name=form novalidate)
+                button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
 
-            .ui-grid-settings-dateperiod
-                +ignite-form-field-datepicker('Period:', '$ctrl.params.startDate', '"period"')
+                .ui-grid-settings-dateperiod
+                    +ignite-form-field-datepicker('Period:', '$ctrl.params.startDate', '"period"', null, '$ctrl.params.endDate')
+                    +ignite-form-field-datepicker('Period:', '$ctrl.params.endDate', null, '$ctrl.params.startDate', null)
 
-            .ui-grid-settings-filter
-                +ignite-form-field-text('Exclude:', '$ctrl.params.companiesExclude', '"exclude"', false, false, 'Exclude by company name...')
+                .ui-grid-settings-filter
+                    +ignite-form-field-text('Exclude:', '$ctrl.params.companiesExclude', '"exclude"', false, false, 'Exclude by company name...')
 
-    .panel-collapse
-        .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-exporter ui-grid-pinning)
+        .panel-collapse
+            .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-exporter ui-grid-pinning ui-grid-grouping)
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
index c6e7bdf241e98..4530c026bb0f2 100644
--- a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.scss
@@ -28,6 +28,12 @@
         height: 30px;
     }
 
+    .ui-grid-header-cell {
+        .ui-grid-cell-contents > span:not(.ui-grid-header-cell-label) {
+            right: 3px;
+        }
+    }
+
     .ui-grid-header-cell [role="columnheader"] {
         display: flex;
         
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug
index 7e44d94671452..9b14fca7f41d6 100644
--- a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug
+++ b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.tpl.pug
@@ -20,8 +20,10 @@
             .ui-grid-header-canvas
                 .ui-grid-header-cell-wrapper(ng-style='colContainer.headerCellWrapperStyle()')
                     .ui-grid-header-cell-row(role='row')
-                        .ui-grid-header-span.ui-grid-header-cell.ui-grid-clearfix(ng-repeat='cat in grid.options.categories')
+                        .ui-grid-header-span.ui-grid-header-cell.ui-grid-clearfix.ui-grid-category(ng-repeat='cat in grid.options.categories', ng-if='cat.visible && \
+                        (colContainer.renderedColumns | uiGridSubcategories: cat.name).length > 0')
                             div(ng-show='(colContainer.renderedColumns|uiGridSubcategories:cat.name).length > 1')
                                 .ui-grid-cell-contents {{ cat.name }}
                             .ui-grid-header-cell-row
                                 .ui-grid-header-cell.ui-grid-clearfix(ng-repeat='col in (colContainer.renderedColumns|uiGridSubcategories:cat.name) track by col.uid' ui-grid-header-cell='' col='col' render-index='$index')
+                        .ui-grid-header-cell.ui-grid-clearfix(ng-if='col.colDef.name === "treeBaseRowHeaderCol"' ng-repeat='col in colContainer.renderedColumns track by col.uid' ui-grid-header-cell='' col='col' render-index='$index')
diff --git a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
index 24f4d9b7efb7f..d0a31f05f464c 100644
--- a/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-settings/ui-grid-settings.scss
@@ -129,6 +129,16 @@
                     width: 60%;
                 }
             }
+
+            &:nth-child(2) {
+                float: left;
+
+                width: 100px;
+
+                .ignite-form-field__control {
+                    width: 100%;
+                }
+            }
         }
     }
 }
diff --git a/modules/web-console/frontend/app/primitives/badge/index.scss b/modules/web-console/frontend/app/primitives/badge/index.scss
new file mode 100644
index 0000000000000..837ab5bc9cad6
--- /dev/null
+++ b/modules/web-console/frontend/app/primitives/badge/index.scss
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@import '../../../public/stylesheets/variables';
+
+.badge {
+  display: inline-block;
+  min-width: 26px;
+  height: 18px;
+
+  padding: 2px 9px;
+
+  border-radius: 9px;
+
+  color: white;
+  font-family: Roboto;
+  font-size: 12px;
+  text-align: center;
+  line-height: 12px;
+
+  background-color: $brand-primary;
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/primitives/index.js b/modules/web-console/frontend/app/primitives/index.js
new file mode 100644
index 0000000000000..7940f7a788249
--- /dev/null
+++ b/modules/web-console/frontend/app/primitives/index.js
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './badge/index.scss';
+import './tabs/index.scss';
diff --git a/modules/web-console/frontend/app/primitives/tabs/index.scss b/modules/web-console/frontend/app/primitives/tabs/index.scss
new file mode 100644
index 0000000000000..eed88cb0480c4
--- /dev/null
+++ b/modules/web-console/frontend/app/primitives/tabs/index.scss
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@import '../../../public/stylesheets/variables';
+
+ul.tabs {
+    $width: auto;
+    $height: 40px;
+    $offset-vertical: 11px;
+    $offset-horizontal: 25px;
+    $font-size: 14px;
+
+    list-style: none;
+
+    padding-left: 0;
+    border-bottom: 1px solid $nav-tabs-border-color;
+
+    li {
+        position: relative;
+        top: 1px;
+
+        display: inline-block;
+
+        border-bottom: 5px solid transparent;
+
+        a {
+            display: inline-block;
+            width: $width;
+            height: $height;
+
+            padding: $offset-vertical $offset-horizontal;
+
+            color: $text-color;
+            font-size: $font-size;
+            text-align: center;
+            line-height: $height - 2*$offset-vertical;
+
+            &:hover {
+              text-decoration: none;
+            }
+
+            .badge {
+              margin-left: $offset-vertical;
+            }
+        }
+
+        &.active {
+            border-color: $brand-primary;
+        }
+
+        &:not(.active):hover {
+            border-color: lighten($brand-primary, 25%);
+        }
+
+        & + li {
+            margin-left: 45px;
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index a07472e8840a4..2f4966fc8170c 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -2302,6 +2302,8 @@ html,body,.splash-screen {
 
 .admin-page {
     .panel-heading {
+        height: 38px;
+
         border-bottom: 0;
         padding-bottom: 0;
 

From 7d94963251cc66823883d760668c2e2b31574bf1 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 22 Mar 2017 18:20:32 +0700
Subject: [PATCH 072/516] IGNITE-4659 Fixed error page.

(cherry picked from commit 50de012)
---
 .../web-console/frontend/app/modules/states/errors.state.js   | 4 ++--
 modules/web-console/frontend/views/{403.pug => 403.tpl.pug}   | 0
 modules/web-console/frontend/views/{404.pug => 404.tpl.pug}   | 0
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename modules/web-console/frontend/views/{403.pug => 403.tpl.pug} (100%)
 rename modules/web-console/frontend/views/{404.pug => 404.tpl.pug} (100%)

diff --git a/modules/web-console/frontend/app/modules/states/errors.state.js b/modules/web-console/frontend/app/modules/states/errors.state.js
index e219132beb34e..e816ff80224ae 100644
--- a/modules/web-console/frontend/app/modules/states/errors.state.js
+++ b/modules/web-console/frontend/app/modules/states/errors.state.js
@@ -16,8 +16,8 @@
  */
 
 import angular from 'angular';
-import templateNotFoundPage from 'views/404.pug';
-import templateNotAuthorizedPage from 'views/403.pug';
+import templateNotFoundPage from 'views/404.tpl.pug';
+import templateNotAuthorizedPage from 'views/403.tpl.pug';
 
 angular
     .module('ignite-console.states.errors', [
diff --git a/modules/web-console/frontend/views/403.pug b/modules/web-console/frontend/views/403.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/403.pug
rename to modules/web-console/frontend/views/403.tpl.pug
diff --git a/modules/web-console/frontend/views/404.pug b/modules/web-console/frontend/views/404.tpl.pug
similarity index 100%
rename from modules/web-console/frontend/views/404.pug
rename to modules/web-console/frontend/views/404.tpl.pug

From eb837c7853ffc68e5a3f690e985407e603e3f955 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Wed, 22 Mar 2017 18:21:03 +0700
Subject: [PATCH 073/516] IGNITE-4854 Fixed project structure button on summary
 page under firefox.

(cherry picked from commit 117e18e)
---
 .../web-console/frontend/views/configuration/summary.tpl.pug  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/frontend/views/configuration/summary.tpl.pug b/modules/web-console/frontend/views/configuration/summary.tpl.pug
index 9fdd0de3f3097..13d6e3a2c79da 100644
--- a/modules/web-console/frontend/views/configuration/summary.tpl.pug
+++ b/modules/web-console/frontend/views/configuration/summary.tpl.pug
@@ -43,8 +43,8 @@ mixin hard-link(ref, txt)
                             i.fa.fa-fw.fa-download(ng-hide='isPrepareDownloading')
                             i.fa.fa-fw.fa-refresh.fa-spin(ng-show='isPrepareDownloading')
                             span.tipLabel Download project
-                    button.btn.btn-primary(bs-tooltip='' data-title='Preview generated project structure' data-placement='bottom')
-                        div(bs-popover data-template-url='{{ ctrl.summaryProjectStructureTemplateUrl }}', data-placement='bottom', data-trigger='click' data-auto-close='true')
+                    span(bs-tooltip='' data-title='Preview generated project structure' data-placement='bottom')
+                        button.btn.btn-primary(bs-popover data-template-url='{{ ctrl.summaryProjectStructureTemplateUrl }}', data-placement='bottom', data-trigger='click' data-auto-close='true')
                             i.fa.fa-sitemap
                             label.tipLabel Project structure
                     button.btn.btn-primary(id='proprietary-jdbc-drivers' ng-if='downloadJdbcDriversVisible()' ng-click='downloadJdbcDrivers()' bs-tooltip='' data-title='Open proprietary JDBC drivers download pages' data-placement='bottom') Download JDBC drivers

From ec7b9d848274b229b69dc7b8a20902654f719c44 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 23 Mar 2017 09:32:38 +0700
Subject: [PATCH 074/516] IGNITE-4855 Fixed error on switching between
 notebooks.

(cherry picked from commit 6a148e2)
---
 .../frontend/app/modules/sql/sql.module.js           | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/sql/sql.module.js b/modules/web-console/frontend/app/modules/sql/sql.module.js
index 670e4f45090d9..79614c2c9c73d 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.module.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.module.js
@@ -34,9 +34,7 @@ angular.module('ignite-console.sql', [
                 .state('base.sql', {
                     url: '/queries',
                     abstract: true,
-                    template: '<ui-view></ui-view>',
-                    controller,
-                    controllerAs: '$ctrl'
+                    template: '<ui-view></ui-view>'
                 })
                 .state('base.sql.notebook', {
                     url: '/notebook/{noteId}',
@@ -44,7 +42,9 @@ angular.module('ignite-console.sql', [
                     onEnter: AclRoute.checkAccess('query'),
                     metaTags: {
                         title: 'Query notebook'
-                    }
+                    },
+                    controller,
+                    controllerAs: '$ctrl'
                 })
                 .state('base.sql.demo', {
                     url: '/demo',
@@ -52,7 +52,9 @@ angular.module('ignite-console.sql', [
                     onEnter: AclRoute.checkAccess('query'),
                     metaTags: {
                         title: 'SQL demo'
-                    }
+                    },
+                    controller,
+                    controllerAs: '$ctrl'
                 });
         }]
     )

From 36f7621b6eaa710d3b1eba7fddd0dfe92e11133e Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Thu, 23 Mar 2017 10:57:03 +0700
Subject: [PATCH 075/516] IGNITE-4830 Fixed error ui.

(cherry picked from commit 48e78a9)
---
 .../frontend/public/stylesheets/style.scss      |  3 +--
 .../web-console/frontend/views/sql/sql.tpl.pug  | 17 +++++++++--------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index 2f4966fc8170c..8dbf123f17688 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -752,8 +752,7 @@ button.form-control {
         .error {
             padding: 10px 10px;
 
-            text-align: center;
-            color: $brand-primary;
+            text-align: left;
         }
 
         .empty {
diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug
index 701ee143af0ca..dcfc53137f616 100644
--- a/modules/web-console/frontend/views/sql/sql.tpl.pug
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -110,10 +110,9 @@ mixin query-settings
                     span Allow non-collocated joins
             .row(ng-if='enforceJoinOrderAvailable(paragraph)')
                 label.tipLabel(bs-tooltip data-placement='bottom' data-title='Enforce join order of tables in the query.<br/>\
-                    If <b>set</b> then query optimizer will not reorder tables in join.<br/>\
-                    <b>NOTE:</b> It is not recommended to enable this property until you are sure that\
-                    your indexes and the query itself are correct and tuned as much as possible but\
-                    query optimizer still produces wrong join order.' data-trigger='hover')
+                    If <b>set</b>, then query optimizer will not reorder tables within join.<br/>\
+                    <b>NOTE:</b> It is not recommended to enable this property unless you have verified that\
+                    indexes are not selected in optimal order.' data-trigger='hover')
                     input(type='checkbox' ng-model='paragraph.enforceJoinOrder')
                     span Enforce join order
 
@@ -247,16 +246,18 @@ mixin paragraph-query
             .pull-right
                 +query-settings
         .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
-            .error(ng-switch-when='error') Error: {{paragraph.error.message}}
+            .error(ng-switch-when='error')
+                label Error: {{paragraph.error.message}}
+                br
+                a(ng-show='paragraph.resultType() === "error"' ng-click='showStackTrace(paragraph)') Show more
             .empty(ng-switch-when='empty') Result set is empty
             .table(ng-switch-when='table')
                 +table-result-heading-query
                 +table-result-body
             .chart(ng-switch-when='chart')
                 +chart-result
-            .footer.clearfix
-                a.pull-left(ng-show='paragraph.resultType() === "error"' ng-click='showStackTrace(paragraph)') Show error details
-                a.pull-left(ng-show='paragraph.resultType() !== "error"' ng-click='showResultQuery(paragraph)') Show query
+            .footer.clearfix(ng-show='paragraph.resultType() !== "error"')
+                a.pull-left(ng-click='showResultQuery(paragraph)') Show query
 
                 -var nextVisibleCondition = 'paragraph.resultType() !== "error" && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())'
 

From 1f3b2fcd003c1f084874d5c421953da0a7cd02cb Mon Sep 17 00:00:00 2001
From: Valentin Kulichenko <valentin.lulichenko@gmail.com>
Date: Mon, 27 Mar 2017 18:12:17 -0700
Subject: [PATCH 076/516] IGNITE-4802 - Separate thread pool for managed
 services

---
 .../configuration/IgniteConfiguration.java    |  30 ++++
 .../ignite/internal/GridKernalContext.java    |   7 +
 .../internal/GridKernalContextImpl.java       |  11 ++
 .../apache/ignite/internal/IgniteKernal.java  |   2 +
 .../apache/ignite/internal/IgnitionEx.java    |  16 +++
 .../managers/communication/GridIoManager.java |   2 +
 .../managers/communication/GridIoPolicy.java  |   5 +-
 .../processors/pool/PoolProcessor.java        |  10 +-
 .../processors/service/GridServiceProxy.java  |  10 ++
 .../task/GridTaskThreadContextKey.java        |   7 +-
 .../processors/task/GridTaskWorker.java       |  16 ++-
 .../services/ServiceThreadPoolSelfTest.java   | 133 ++++++++++++++++++
 .../junits/GridTestKernalContext.java         |   1 +
 .../testsuites/IgniteKernalSelfTestSuite.java |   2 +
 14 files changed, 245 insertions(+), 7 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/services/ServiceThreadPoolSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index dcd8a801aa808..b240446cd5ae0 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -53,6 +53,7 @@
 import org.apache.ignite.plugin.PluginProvider;
 import org.apache.ignite.plugin.segmentation.SegmentationPolicy;
 import org.apache.ignite.plugin.segmentation.SegmentationResolver;
+import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.spi.checkpoint.CheckpointSpi;
 import org.apache.ignite.spi.checkpoint.noop.NoopCheckpointSpi;
@@ -233,6 +234,9 @@ public class IgniteConfiguration {
     /** Public pool size. */
     private int pubPoolSize = DFLT_PUBLIC_THREAD_CNT;
 
+    /** Service pool size. */
+    private Integer svcPoolSize;
+
     /** Async Callback pool size. */
     private int callbackPoolSize = DFLT_PUBLIC_THREAD_CNT;
 
@@ -561,6 +565,7 @@ public IgniteConfiguration(IgniteConfiguration cfg) {
         storeSesLsnrs = cfg.getCacheStoreSessionListenerFactories();
         stripedPoolSize = cfg.getStripedPoolSize();
         svcCfgs = cfg.getServiceConfiguration();
+        svcPoolSize = cfg.getServiceThreadPoolSize();
         sysPoolSize = cfg.getSystemThreadPoolSize();
         timeSrvPortBase = cfg.getTimeServerPortBase();
         timeSrvPortRange = cfg.getTimeServerPortRange();
@@ -773,6 +778,18 @@ public int getPublicThreadPoolSize() {
         return pubPoolSize;
     }
 
+    /**
+     * Should return a thread pool size to be used in grid.
+     * This executor service will be in charge of processing {@link Service} proxy invocations.
+     * <p>
+     * If not provided, executor service will have size {@link #DFLT_PUBLIC_THREAD_CNT}.
+     *
+     * @return Thread pool size to be used in grid to process service proxy invocations.
+     */
+    public int getServiceThreadPoolSize() {
+        return svcPoolSize != null ? svcPoolSize : getPublicThreadPoolSize();
+    }
+
     /**
      * Size of thread pool that is in charge of processing internal system messages.
      * <p>
@@ -893,6 +910,19 @@ public IgniteConfiguration setPublicThreadPoolSize(int poolSize) {
         return this;
     }
 
+    /**
+     * Sets thread pool size to use within grid.
+     *
+     * @param poolSize Thread pool size to use within grid.
+     * @see IgniteConfiguration#getServiceThreadPoolSize()
+     * @return {@code this} for chaining.
+     */
+    public IgniteConfiguration setServiceThreadPoolSize(int poolSize) {
+        svcPoolSize = poolSize;
+
+        return this;
+    }
+
     /**
      * Sets system thread pool size to use within grid.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 927944fd908c5..d95f09cce3e17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -504,6 +504,13 @@ public interface GridKernalContext extends Iterable<GridComponent> {
      */
     public ExecutorService getExecutorService();
 
+    /**
+     * Executor service that is in charge of processing service proxy invocations.
+     *
+     * @return Thread pool implementation to be used in grid for service proxy invocations.
+     */
+    public ExecutorService getServiceExecutorService();
+
     /**
      * Executor service that is in charge of processing internal system messages.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index a2ad1b2043368..8e1fd7e9eb049 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -295,6 +295,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     @GridToStringExclude
     protected ExecutorService execSvc;
 
+    /** */
+    @GridToStringExclude
+    protected ExecutorService svcExecSvc;
+
     /** */
     @GridToStringExclude
     protected ExecutorService sysExecSvc;
@@ -405,6 +409,7 @@ protected GridKernalContextImpl(
         ExecutorService utilityCachePool,
         ExecutorService marshCachePool,
         ExecutorService execSvc,
+        ExecutorService svcExecSvc,
         ExecutorService sysExecSvc,
         StripedExecutor stripedExecSvc,
         ExecutorService p2pExecSvc,
@@ -426,6 +431,7 @@ protected GridKernalContextImpl(
         this.utilityCachePool = utilityCachePool;
         this.marshCachePool = marshCachePool;
         this.execSvc = execSvc;
+        this.svcExecSvc = svcExecSvc;
         this.sysExecSvc = sysExecSvc;
         this.stripedExecSvc = stripedExecSvc;
         this.p2pExecSvc = p2pExecSvc;
@@ -951,6 +957,11 @@ protected Object readResolve() throws ObjectStreamException {
         return execSvc;
     }
 
+    /** {@inheritDoc} */
+    @Override public ExecutorService getServiceExecutorService() {
+        return svcExecSvc;
+    }
+
     /** {@inheritDoc} */
     @Override public ExecutorService getSystemExecutorService() {
         return sysExecSvc;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 063fe255af95f..716116625bc6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -686,6 +686,7 @@ public void start(
         ExecutorService utilityCachePool,
         ExecutorService marshCachePool,
         final ExecutorService execSvc,
+        final ExecutorService svcExecSvc,
         final ExecutorService sysExecSvc,
         final StripedExecutor stripedExecSvc,
         ExecutorService p2pExecSvc,
@@ -795,6 +796,7 @@ public void start(
                 utilityCachePool,
                 marshCachePool,
                 execSvc,
+                svcExecSvc,
                 sysExecSvc,
                 stripedExecSvc,
                 p2pExecSvc,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index f32a75309166c..8212dd0512de7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -1457,6 +1457,9 @@ private static final class IgniteNamedInstance {
         /** Executor service. */
         private ThreadPoolExecutor execSvc;
 
+        /** Executor service for services. */
+        private ThreadPoolExecutor svcExecSvc;
+
         /** System executor service. */
         private ThreadPoolExecutor sysExecSvc;
 
@@ -1656,6 +1659,18 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
 
             execSvc.allowCoreThreadTimeOut(true);
 
+            validateThreadPoolSize(cfg.getServiceThreadPoolSize(), "service");
+
+            svcExecSvc = new IgniteThreadPoolExecutor(
+                "svc",
+                cfg.getGridName(),
+                cfg.getServiceThreadPoolSize(),
+                cfg.getServiceThreadPoolSize(),
+                DFLT_THREAD_KEEP_ALIVE_TIME,
+                new LinkedBlockingQueue<Runnable>());
+
+            svcExecSvc.allowCoreThreadTimeOut(true);
+
             validateThreadPoolSize(cfg.getSystemThreadPoolSize(), "system");
 
             sysExecSvc = new IgniteThreadPoolExecutor(
@@ -1801,6 +1816,7 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                     utilityCacheExecSvc,
                     marshCacheExecSvc,
                     execSvc,
+                    svcExecSvc,
                     sysExecSvc,
                     stripedExecSvc,
                     p2pExecSvc,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 7ef7bc080fe98..2eda4b715ee01 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -96,6 +96,7 @@
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.MARSH_CACHE_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.P2P_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.PUBLIC_POOL;
+import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SERVICE_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.UTILITY_CACHE_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.isReservedGridIoPolicy;
@@ -686,6 +687,7 @@ private void onMessage0(UUID nodeId, GridIoMessage msg, IgniteRunnable msgC) {
                 case MARSH_CACHE_POOL:
                 case IDX_POOL:
                 case IGFS_POOL:
+                case SERVICE_POOL:
                 {
                     if (msg.isOrdered())
                         processOrderedMessage(nodeId, msg, plc, msgC);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java
index 70a73541d2ffa..bb64f6a083acf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java
@@ -49,6 +49,9 @@ public class GridIoPolicy {
     /** Pool for handling distributed index range requests. */
     public static final byte IDX_POOL = 8;
 
+    /** Pool for service proxy executions. */
+    public static final byte SERVICE_POOL = 9;
+
     /**
      * Defines the range of reserved pools that are not available for plugins.
      * @param key The key.
@@ -57,4 +60,4 @@ public class GridIoPolicy {
     public static boolean isReservedGridIoPolicy(byte key) {
         return key >= 0 && key <= 31;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
index 59e5e7dffb7e9..66efc8c2084b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.pool;
 
+import java.util.Arrays;
+import java.util.concurrent.Executor;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.GridKernalContext;
@@ -25,9 +27,6 @@
 import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor;
 import org.apache.ignite.plugin.extensions.communication.IoPool;
 
-import java.util.Arrays;
-import java.util.concurrent.Executor;
-
 /**
  * Processor which abstracts out thread pool management.
  */
@@ -128,6 +127,11 @@ public Executor poolForPolicy(byte plc) throws IgniteCheckedException {
 
                 return ctx.getIgfsExecutorService();
 
+            case GridIoPolicy.SERVICE_POOL:
+                assert ctx.getServiceExecutorService() != null : "Service pool is not configured.";
+
+                return ctx.getServiceExecutorService();
+
             default: {
                 if (plc < 0)
                     throw new IgniteCheckedException("Policy cannot be negative: " + plc);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index aa609340b3bfb..d2e96bac52c0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -42,15 +42,19 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.services.Service;
 import org.jsr166.ThreadLocalRandom8;
 
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_IO_POLICY;
+
 /**
  * Wrapper for making {@link org.apache.ignite.services.Service} class proxies.
  */
@@ -58,6 +62,9 @@ public class GridServiceProxy<T> implements Serializable {
     /** */
     private static final long serialVersionUID = 0L;
 
+    /** */
+    private static final IgniteProductVersion SVC_POOL_SINCE_VER = IgniteProductVersion.fromString("1.8.5");
+
     /** Grid logger. */
     @GridToStringExclude
     private final IgniteLogger log;
@@ -176,6 +183,9 @@ else if (U.isToStringMethod(mtd))
                         }
                     }
                     else {
+                        if (node.version().compareTo(SVC_POOL_SINCE_VER) >= 0)
+                            ctx.task().setThreadContext(TC_IO_POLICY, GridIoPolicy.SERVICE_POOL);
+
                         // Execute service remotely.
                         return ctx.closure().callAsyncNoFailover(
                             GridClosureCallMode.BROADCAST,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
index 3bb19241a4ebf..2ec63df4ed83c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
@@ -34,5 +34,8 @@ public enum GridTaskThreadContextKey {
     TC_TIMEOUT,
 
     /** Security subject ID. */
-    TC_SUBJ_ID
-}
\ No newline at end of file
+    TC_SUBJ_ID,
+
+    /** IO manager policy. */
+    TC_IO_POLICY
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
index d89e80bc99825..c426008f5fb11 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
@@ -101,6 +101,7 @@
 import static org.apache.ignite.internal.GridTopic.TOPIC_JOB_CANCEL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.MANAGEMENT_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.PUBLIC_POOL;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_IO_POLICY;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
 
 /**
@@ -1381,8 +1382,21 @@ private void sendRequest(ComputeJobResult res) {
                     if (loc)
                         ctx.job().processJobExecuteRequest(ctx.discovery().localNode(), req);
                     else {
+                        byte plc;
+
+                        if (internal)
+                            plc = MANAGEMENT_POOL;
+                        else {
+                            Byte ctxPlc = getThreadContext(TC_IO_POLICY);
+
+                            if (ctxPlc != null)
+                                plc = ctxPlc;
+                            else
+                                plc = PUBLIC_POOL;
+                        }
+
                         // Send job execution request.
-                        ctx.io().send(node, TOPIC_JOB, req, internal ? MANAGEMENT_POOL : PUBLIC_POOL);
+                        ctx.io().send(node, TOPIC_JOB, req, plc);
 
                         if (log.isDebugEnabled())
                             log.debug("Sent job request [req=" + req + ", node=" + node + ']');
diff --git a/modules/core/src/test/java/org/apache/ignite/services/ServiceThreadPoolSelfTest.java b/modules/core/src/test/java/org/apache/ignite/services/ServiceThreadPoolSelfTest.java
new file mode 100644
index 0000000000000..40efdfaa91d4f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/services/ServiceThreadPoolSelfTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.services;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test verifying that services thread pool is properly used.
+ */
+public class ServiceThreadPoolSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDefaultPoolSize() throws Exception {
+        Ignite ignite = startGrid("grid", new IgniteConfiguration());
+
+        IgniteConfiguration cfg = ignite.configuration();
+
+        assertEquals(IgniteConfiguration.DFLT_PUBLIC_THREAD_CNT, cfg.getPublicThreadPoolSize());
+        assertEquals(IgniteConfiguration.DFLT_PUBLIC_THREAD_CNT, cfg.getServiceThreadPoolSize());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testInheritedPoolSize() throws Exception {
+        Ignite ignite = startGrid("grid", new IgniteConfiguration().setPublicThreadPoolSize(42));
+
+        IgniteConfiguration cfg = ignite.configuration();
+
+        assertEquals(42, cfg.getPublicThreadPoolSize());
+        assertEquals(42, cfg.getServiceThreadPoolSize());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCustomPoolSize() throws Exception {
+        Ignite ignite = startGrid("grid", new IgniteConfiguration().setServiceThreadPoolSize(42));
+
+        IgniteConfiguration cfg = ignite.configuration();
+
+        assertEquals(IgniteConfiguration.DFLT_PUBLIC_THREAD_CNT, cfg.getPublicThreadPoolSize());
+        assertEquals(42, cfg.getServiceThreadPoolSize());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testExecution() throws Exception {
+        startGrid(0); // Server.
+
+        Ignition.setClientMode(true);
+
+        Ignite ignite = startGrid(); // Client.
+
+        ignite.services().deployClusterSingleton("my-service", new MyServiceImpl());
+
+        MyService svc = ignite.services().serviceProxy("my-service", MyService.class, false);
+
+        svc.hello();
+    }
+
+    /**
+     */
+    private interface MyService extends Service{
+        /**
+         * Hello!
+         */
+        void hello();
+    }
+
+    /**
+     */
+    private static class MyServiceImpl implements MyService {
+        /** {@inheritDoc} */
+        @Override public void hello() {
+            String thread = Thread.currentThread().getName();
+
+            assertTrue("Service is executed in wrong thread: " + thread, thread.startsWith("svc-#"));
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(ServiceContext ctx) throws Exception {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
index 143159dd4c57f..49cb206ee3fb3 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
@@ -64,6 +64,7 @@ public GridTestKernalContext(IgniteLogger log, IgniteConfiguration cfg) throws I
                 null,
                 null,
                 null,
+                null,
                 U.allPluginProviders()
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 4f771d6529b85..9b41fbcf96873 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -72,6 +72,7 @@
 import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
 import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
 import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
+import org.apache.ignite.services.ServiceThreadPoolSelfTest;
 import org.apache.ignite.spi.communication.GridCacheMessageSelfTest;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -147,6 +148,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(IgniteServiceProxyTimeoutInitializedTest.class);
         suite.addTestSuite(IgniteServiceDynamicCachesSelfTest.class);
         suite.addTestSuite(GridServiceContinuousQueryRedeploy.class);
+        suite.addTestSuite(ServiceThreadPoolSelfTest.class);
 
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class);

From a54b7c6d76974f833e451aff947456ea0013226b Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 28 Mar 2017 18:49:03 +0300
Subject: [PATCH 077/516] IGNITE-4826: Fix eviction to swap if segmented index
 is used. This closes #1628.

---
 .../query/h2/opt/GridH2TreeIndex.java         |  4 +--
 .../IgniteSqlSegmentedIndexSelfTest.java      | 30 +++++++++++++++++--
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
index 2873211fdf175..663d86386b8de 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
@@ -286,8 +286,8 @@ protected ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> treeForRead(in
      * @param row Search row.
      * @return Row.
      */
-    public GridH2Row findOne(GridSearchRowPointer row) {
-        int seg = threadLocalSegment();
+    GridH2Row findOne(GridSearchRowPointer row) {
+        int seg = segmentForRow(row);
 
         return segments[seg].get(row);
     }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
index f8c9dd5b64de2..800138c1f32ac 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheKeyConfiguration;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -34,6 +35,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.swapspace.file.FileSwapSpaceSpi;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -63,6 +65,8 @@ public class IgniteSqlSegmentedIndexSelfTest extends GridCommonAbstractTest {
 
         cfg.setDiscoverySpi(disco);
 
+        cfg.setSwapSpaceSpi(new FileSwapSpaceSpi());
+
         return cfg;
     }
 
@@ -77,7 +81,7 @@ public class IgniteSqlSegmentedIndexSelfTest extends GridCommonAbstractTest {
      * @param idxTypes Indexed types.
      * @return Cache configuration.
      */
-    private static <K, V> CacheConfiguration<K, V> cacheConfig(String name, boolean partitioned, Class<?>... idxTypes) {
+    protected <K, V> CacheConfiguration<K, V> cacheConfig(String name, boolean partitioned, Class<?>... idxTypes) {
         return new CacheConfiguration<K, V>()
             .setName(name)
             .setCacheMode(partitioned ? CacheMode.PARTITIONED : CacheMode.REPLICATED)
@@ -103,6 +107,28 @@ public void testSingleNodeIndexSegmentation() throws Exception {
         checkLocalQueryWithSegmentedIndex();
     }
 
+    /**
+     * Run tests on single-node grid
+     * @throws Exception If failed.
+     */
+    public void testSingleNodeIndexSegmentationWithSwapEnabled() throws Exception {
+        startGridsMultiThreaded(1, true);
+
+        final IgniteCache<Object, Object> cache = ignite(0).createCache(cacheConfig("org", true, Integer.class, Organization.class)
+            .setOffHeapMaxMemory(-1)
+            .setSwapEnabled(true)
+            .setEvictionPolicy(new FifoEvictionPolicy(10)));
+
+        for (int i = 0; i < 20; i++)
+            cache.put(i, new Organization("org-" + i));
+
+        String select0 = "select name from \"org\".Organization";
+
+        List<List<?>> result = cache.query(new SqlFieldsQuery(select0)).getAll();
+
+        assertEquals(20, result.size());
+    }
+
     /**
      * Run tests on multi-node grid
      * @throws Exception If failed.
@@ -170,7 +196,7 @@ public void checkLocalQueryWithSegmentedIndex() throws Exception {
 
         Set<Integer> localOrgIds = new HashSet<>();
 
-        for(Cache.Entry<Integer, Organization> e : c2.localEntries())
+        for (Cache.Entry<Integer, Organization> e : c2.localEntries())
             localOrgIds.add(e.getKey());
 
         int expectedPersons = 0;

From 39edcc7890ce497771f532a83a57ecae318d8c88 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 28 Mar 2017 18:56:17 +0300
Subject: [PATCH 078/516] IGNITE-4815: Fixed confusing javadoc. This closes
 #1613.

---
 .../org/apache/ignite/spi/collision/CollisionSpi.java | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/collision/CollisionSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/collision/CollisionSpi.java
index d82d9a1dccb34..9b191f6a5d220 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/collision/CollisionSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/collision/CollisionSpi.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.spi.collision;
 
+import org.apache.ignite.events.EventType;
 import org.apache.ignite.spi.IgniteSpi;
 import org.jetbrains.annotations.Nullable;
 
@@ -48,8 +49,14 @@
  */
 public interface CollisionSpi extends IgniteSpi {
     /**
-     * This is a callback called when either new grid job arrived or executing job finished its
-     * execution. When new job arrives it is added to the end of the wait list and this
+     * This is a callback called:
+     * <ul>
+     *      <li>new grid job arrived</li>
+     *      <li>executing job finished its execution</li>
+     *      <li>topology changed</li>
+     *     <li>periodically (on {@link EventType#EVT_NODE_METRICS_UPDATED})</li>
+     * </ul>
+     * When new job arrives it is added to the end of the wait list and this
      * method is called. When job finished its execution, it is removed from the active list and
      * this method is called (i.e., when grid job is finished it will not appear in any list
      * in collision resolution).

From e47bf27b5582cd695a7d3de3c39d54b3df4c59cc Mon Sep 17 00:00:00 2001
From: Valentin Kulichenko <valentin.lulichenko@gmail.com>
Date: Tue, 28 Mar 2017 16:51:44 -0700
Subject: [PATCH 079/516] IGNITE-4762 - transactionsAllowed flag for JDBC
 driver

---
 .../jdbc2/JdbcConnectionSelfTest.java         | 34 +++++++++++++
 .../org/apache/ignite/IgniteJdbcDriver.java   |  9 +++-
 .../ignite/internal/jdbc2/JdbcConnection.java | 50 ++++++++++++-------
 3 files changed, 75 insertions(+), 18 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java
index 8c05df3adf700..0bbdca86c67f9 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java
@@ -267,4 +267,38 @@ public void testClose() throws Exception {
             );
         }
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxAllowedCommit() throws Exception {
+        String url = CFG_URL_PREFIX + "transactionsAllowed=true@" + CFG_URL;
+
+        try (final Connection conn = DriverManager.getConnection(url)) {
+            conn.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
+
+            assertEquals(Connection.TRANSACTION_SERIALIZABLE, conn.getTransactionIsolation());
+
+            conn.setAutoCommit(false);
+
+            conn.commit();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxAllowedRollback() throws Exception {
+        String url = CFG_URL_PREFIX + "transactionsAllowed=true@" + CFG_URL;
+
+        try (final Connection conn = DriverManager.getConnection(url)) {
+            conn.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
+
+            assertEquals(Connection.TRANSACTION_SERIALIZABLE, conn.getTransactionIsolation());
+
+            conn.setAutoCommit(false);
+
+            conn.rollback();
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
index d432c1e4f9fee..c511d5f8b7c02 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
@@ -292,6 +292,9 @@ public class IgniteJdbcDriver implements Driver {
     /** Distributed joins parameter name. */
     private static final String PARAM_DISTRIBUTED_JOINS = "distributedJoins";
 
+    /** Transactions allowed parameter name. */
+    private static final String PARAM_TX_ALLOWED = "transactionsAllowed";
+
     /** Hostname property name. */
     public static final String PROP_HOST = PROP_PREFIX + "host";
 
@@ -313,6 +316,9 @@ public class IgniteJdbcDriver implements Driver {
     /** Distributed joins property name. */
     public static final String PROP_DISTRIBUTED_JOINS = PROP_PREFIX + PARAM_DISTRIBUTED_JOINS;
 
+    /** Transactions allowed property name. */
+    public static final String PROP_TX_ALLOWED = PROP_PREFIX + PARAM_TX_ALLOWED;
+
     /** Cache name property name. */
     public static final String PROP_CFG = PROP_PREFIX + "cfg";
 
@@ -378,7 +384,8 @@ public class IgniteJdbcDriver implements Driver {
             new PropertyInfo("Node ID", info.getProperty(PROP_NODE_ID), ""),
             new PropertyInfo("Local", info.getProperty(PROP_LOCAL), ""),
             new PropertyInfo("Collocated", info.getProperty(PROP_COLLOCATED), ""),
-            new PropertyInfo("Distributed Joins", info.getProperty(PROP_DISTRIBUTED_JOINS), "")
+            new PropertyInfo("Distributed Joins", info.getProperty(PROP_DISTRIBUTED_JOINS), ""),
+            new PropertyInfo("Transactions Allowed", info.getProperty(PROP_TX_ALLOWED), "")
         );
 
         if (info.getProperty(PROP_CFG) != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
index 5c4a147174b45..dc3fe7f454624 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
@@ -73,6 +73,7 @@
 import static org.apache.ignite.IgniteJdbcDriver.PROP_DISTRIBUTED_JOINS;
 import static org.apache.ignite.IgniteJdbcDriver.PROP_LOCAL;
 import static org.apache.ignite.IgniteJdbcDriver.PROP_NODE_ID;
+import static org.apache.ignite.IgniteJdbcDriver.PROP_TX_ALLOWED;
 
 /**
  * JDBC connection implementation.
@@ -118,6 +119,12 @@ public class JdbcConnection implements Connection {
     /** Distributed joins flag. */
     private boolean distributedJoins;
 
+    /** Transactions allowed flag. */
+    private boolean txAllowed;
+
+    /** Current transaction isolation. */
+    private int txIsolation;
+
     /** Statements. */
     final Set<JdbcStatement> statements = new HashSet<>();
 
@@ -134,15 +141,16 @@ public JdbcConnection(String url, Properties props) throws SQLException {
 
         this.url = url;
 
-        this.cacheName = props.getProperty(PROP_CACHE);
-        this.locQry = Boolean.parseBoolean(props.getProperty(PROP_LOCAL));
-        this.collocatedQry = Boolean.parseBoolean(props.getProperty(PROP_COLLOCATED));
-        this.distributedJoins = Boolean.parseBoolean(props.getProperty(PROP_DISTRIBUTED_JOINS));
+        cacheName = props.getProperty(PROP_CACHE);
+        locQry = Boolean.parseBoolean(props.getProperty(PROP_LOCAL));
+        collocatedQry = Boolean.parseBoolean(props.getProperty(PROP_COLLOCATED));
+        distributedJoins = Boolean.parseBoolean(props.getProperty(PROP_DISTRIBUTED_JOINS));
+        txAllowed = Boolean.parseBoolean(props.getProperty(PROP_TX_ALLOWED));
 
         String nodeIdProp = props.getProperty(PROP_NODE_ID);
 
         if (nodeIdProp != null)
-            this.nodeId = UUID.fromString(nodeIdProp);
+            nodeId = UUID.fromString(nodeIdProp);
 
         try {
             String cfgUrl = props.getProperty(PROP_CFG);
@@ -259,7 +267,7 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public void setAutoCommit(boolean autoCommit) throws SQLException {
         ensureNotClosed();
 
-        if (!autoCommit)
+        if (!txAllowed && !autoCommit)
             throw new SQLFeatureNotSupportedException("Transactions are not supported.");
     }
 
@@ -274,14 +282,16 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public void commit() throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        if (!txAllowed)
+            throw new SQLFeatureNotSupportedException("Transactions are not supported.");
     }
 
     /** {@inheritDoc} */
     @Override public void rollback() throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        if (!txAllowed)
+            throw new SQLFeatureNotSupportedException("Transactions are not supported.");
     }
 
     /** {@inheritDoc} */
@@ -354,14 +364,20 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public void setTransactionIsolation(int level) throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        if (txAllowed)
+            txIsolation = level;
+        else
+            throw new SQLFeatureNotSupportedException("Transactions are not supported.");
     }
 
     /** {@inheritDoc} */
     @Override public int getTransactionIsolation() throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        if (txAllowed)
+            return txIsolation;
+        else
+            throw new SQLFeatureNotSupportedException("Transactions are not supported.");
     }
 
     /** {@inheritDoc} */
@@ -413,7 +429,7 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public void setHoldability(int holdability) throws SQLException {
         ensureNotClosed();
 
-        if (holdability != HOLD_CURSORS_OVER_COMMIT)
+        if (!txAllowed && holdability != HOLD_CURSORS_OVER_COMMIT)
             throw new SQLFeatureNotSupportedException("Invalid holdability (transactions are not supported).");
     }
 
@@ -428,28 +444,28 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public Savepoint setSavepoint() throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        throw new SQLFeatureNotSupportedException("Savepoints are not supported.");
     }
 
     /** {@inheritDoc} */
     @Override public Savepoint setSavepoint(String name) throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        throw new SQLFeatureNotSupportedException("Savepoints are not supported.");
     }
 
     /** {@inheritDoc} */
     @Override public void rollback(Savepoint savepoint) throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        throw new SQLFeatureNotSupportedException("Savepoints are not supported.");
     }
 
     /** {@inheritDoc} */
     @Override public void releaseSavepoint(Savepoint savepoint) throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("Transactions are not supported.");
+        throw new SQLFeatureNotSupportedException("Savepoints are not supported.");
     }
 
     /** {@inheritDoc} */
@@ -463,7 +479,7 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
         if (resSetConcurrency != CONCUR_READ_ONLY)
             throw new SQLFeatureNotSupportedException("Invalid concurrency (updates are not supported).");
 
-        if (resSetHoldability != HOLD_CURSORS_OVER_COMMIT)
+        if (!txAllowed && resSetHoldability != HOLD_CURSORS_OVER_COMMIT)
             throw new SQLFeatureNotSupportedException("Invalid holdability (transactions are not supported).");
 
         JdbcStatement stmt = new JdbcStatement(this);
@@ -484,7 +500,7 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
         if (resSetConcurrency != CONCUR_READ_ONLY)
             throw new SQLFeatureNotSupportedException("Invalid concurrency (updates are not supported).");
 
-        if (resSetHoldability != HOLD_CURSORS_OVER_COMMIT)
+        if (!txAllowed && resSetHoldability != HOLD_CURSORS_OVER_COMMIT)
             throw new SQLFeatureNotSupportedException("Invalid holdability (transactions are not supported).");
 
         JdbcPreparedStatement stmt = new JdbcPreparedStatement(this, sql);

From 336672432486830c31cb4b6f8bb1b401c276f3e5 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 29 Mar 2017 10:53:25 +0700
Subject: [PATCH 080/516] IGNITE-4659 Fixed typo. (cherry picked from commit
 6f1e970)

---
 .../states/configuration/clusters/general/discovery/s3.pug      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
index ed6e20e4c3c1a..853c32cc5394f 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
@@ -22,6 +22,6 @@ include /app/helpers/jade/mixins
 
 div
     .details-row
-        +text('Bucket name:', `${model}.bucketName`, `${discoveryKind}BucketName'`, required, 'Input bucket name', 'Bucket name for IP finder')
+        +text('Bucket name:', `${model}.bucketName`, `'${discoveryKind}BucketName'`, required, 'Input bucket name', 'Bucket name for IP finder')
     .details-row
         label Note, AWS credentials will be generated as stub

From b689624e2984ec3cf42ca5b01966937e236abcdc Mon Sep 17 00:00:00 2001
From: Alexander Paschenko <alexander.a.paschenko@gmail.com>
Date: Wed, 29 Mar 2017 18:15:20 +0300
Subject: [PATCH 081/516] IGNITE-4868 Raw Object keys and values indexing fix

---
 .../processors/query/GridQueryProcessor.java  |   8 +-
 .../IgniteCacheObjectKeyIndexingSelfTest.java | 124 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite2.java       |   4 +-
 3 files changed, 131 insertions(+), 5 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheObjectKeyIndexingSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 10bf75a01113d..0bde688ded248 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -17,12 +17,11 @@
 
 package org.apache.ignite.internal.processors.query;
 
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.util.concurrent.TimeUnit;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.math.BigDecimal;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
 import java.sql.Time;
 import java.sql.Timestamp;
 import java.util.ArrayList;
@@ -40,6 +39,7 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 import javax.cache.Cache;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
@@ -457,7 +457,7 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
      * @return {@code True} if will be deserialized.
      */
     private boolean mustDeserializeBinary(Class cls) {
-        if (cls != null && ctx.config().getMarshaller() instanceof BinaryMarshaller) {
+        if (cls != null && cls != Object.class && ctx.config().getMarshaller() instanceof BinaryMarshaller) {
             CacheObjectBinaryProcessorImpl proc0 = (CacheObjectBinaryProcessorImpl)ctx.cacheObjects();
 
             return proc0.binaryContext().mustDeserialize(cls);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheObjectKeyIndexingSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheObjectKeyIndexingSelfTest.java
new file mode 100644
index 0000000000000..9e3a0bc160844
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheObjectKeyIndexingSelfTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test index behavior when key is of plain Object type per indexing settings.
+ */
+public class IgniteCacheObjectKeyIndexingSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        Ignition.stopAll(true);
+    }
+
+    /** */
+    protected static CacheConfiguration<Object, TestObject> cacheCfg() {
+        return new CacheConfiguration<Object, TestObject>()
+            .setIndexedTypes(Object.class, TestObject.class);
+    }
+
+    /** */
+    public void testObjectKeyHandling() throws Exception {
+        Ignite ignite = grid();
+
+        IgniteCache<Object, TestObject> cache = ignite.getOrCreateCache(cacheCfg());
+
+        UUID uid = UUID.randomUUID();
+
+        cache.put(uid, new TestObject("A"));
+
+        assertItemsNumber(1);
+
+        cache.put(1, new TestObject("B"));
+
+        assertItemsNumber(2);
+
+        cache.put(uid, new TestObject("C"));
+
+        // Key should have been replaced
+        assertItemsNumber(2);
+
+        List<List<?>> res = cache.query(new SqlFieldsQuery("select _key, name from TestObject order by name")).getAll();
+
+        assertEquals(res,
+            Arrays.asList(
+                Arrays.asList(1, "B"),
+                Arrays.asList(uid, "C")
+            )
+        );
+
+        cache.remove(1);
+
+        assertItemsNumber(1);
+
+        res = cache.query(new SqlFieldsQuery("select _key, name from TestObject")).getAll();
+
+        assertEquals(res,
+            Collections.singletonList(
+                Arrays.asList(uid, "C")
+            )
+        );
+        
+        cache.remove(uid);
+
+        // Removal has worked for both keys although the table was the same and keys were of different type
+        assertItemsNumber(0);
+    }
+
+    /** */
+    @SuppressWarnings("ConstantConditions")
+    private void assertItemsNumber(long num) {
+        assertEquals(num, grid().cachex().size());
+
+        assertEquals(num, grid().cache(null).query(new SqlFieldsQuery("select count(*) from TestObject")).getAll()
+            .get(0).get(0));
+    }
+    
+    /** */
+    private static class TestObject {
+        /** */
+        @QuerySqlField
+        public final String name;
+
+        /** */
+        private TestObject(String name) {
+            this.name = name;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index 8ac219f89f199..923a79c3bd189 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -36,14 +36,15 @@
 import org.apache.ignite.internal.processors.cache.GridCacheQueryIndexingDisabledSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheFieldsQueryNoDataSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheNoClassQuerySelfTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheObjectKeyIndexingSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheP2pUnmarshallingQueryErrorTest;
 import org.apache.ignite.internal.processors.cache.SqlFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheAtomicFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheAtomicNearEnabledFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheDistributedQueryCancelSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheDistributedQueryStopOnCancelOrTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCachePartitionedFieldsQueryP2PEnabledSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCachePartitionedFieldsQuerySelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheDistributedQueryStopOnCancelOrTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryStopOnCancelOrTimeoutDistributedJoinSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedFieldsQueryP2PEnabledSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedFieldsQuerySelfTest;
@@ -120,6 +121,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheQueryNewClientSelfTest.class);
         suite.addTestSuite(CacheOffheapBatchIndexingSingleTypeTest.class);
         suite.addTestSuite(CacheSqlQueryValueCopySelfTest.class);
+        suite.addTestSuite(IgniteCacheObjectKeyIndexingSelfTest.class);
 
         return suite;
     }

From 357c20ab9593390fb7af25f8638188595c5f6cd4 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 30 Mar 2017 12:50:42 +0300
Subject: [PATCH 082/516] IGNITE-4284 - Fix. Failed second client node join
 with continuous query and peer class loading enabled.

---
 .../continuous/GridContinuousProcessor.java   |  39 +++--
 .../custom/DummyEventFilterFactory.java       |  47 ++++++
 .../ContinuousQueryPeerClassLoadingTest.java  | 142 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite3.java       |   2 +
 4 files changed, 214 insertions(+), 16 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 9fd9b6d44b89e..6a4f57d1e3ae8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -67,6 +67,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
@@ -495,29 +496,34 @@ public void unlockStopping() {
             for (Map.Entry<UUID, Map<UUID, LocalRoutineInfo>> entry : data.clientInfos.entrySet()) {
                 UUID clientNodeId = entry.getKey();
 
-                if (!ctx.localNodeId().equals(clientNodeId)) {
+                if (!ctx.clientNode()) {
                     Map<UUID, LocalRoutineInfo> clientRoutineMap = entry.getValue();
 
                     for (Map.Entry<UUID, LocalRoutineInfo> e : clientRoutineMap.entrySet()) {
                         UUID routineId = e.getKey();
                         LocalRoutineInfo info = e.getValue();
 
-                        try {
-                            if (info.prjPred != null)
-                                ctx.resource().injectGeneric(info.prjPred);
-
-                            if (info.prjPred == null || info.prjPred.apply(ctx.discovery().localNode())) {
-                                registerHandler(clientNodeId,
-                                    routineId,
-                                    info.hnd,
-                                    info.bufSize,
-                                    info.interval,
-                                    info.autoUnsubscribe,
-                                    false);
+                        GridCacheContext cctx = ctx.cache().context().cacheContext(CU.cacheId(info.hnd.cacheName()));
+
+                        // Do not register handler if it's not affinity node.
+                        if (cctx == null || cctx.affinityNode()) {
+                            try {
+                                if (info.prjPred != null)
+                                    ctx.resource().injectGeneric(info.prjPred);
+
+                                if (info.prjPred == null || info.prjPred.apply(ctx.discovery().localNode())) {
+                                    registerHandler(clientNodeId,
+                                        routineId,
+                                        info.hnd,
+                                        info.bufSize,
+                                        info.interval,
+                                        info.autoUnsubscribe,
+                                        false);
+                                }
+                            }
+                            catch (IgniteCheckedException err) {
+                                U.error(log, "Failed to register continuous handler.", err);
                             }
-                        }
-                        catch (IgniteCheckedException err) {
-                            U.error(log, "Failed to register continuous handler.", err);
                         }
                     }
                 }
@@ -583,6 +589,7 @@ public void onCacheStop(GridCacheContext ctx) {
      * @return Routine ID.
      * @throws IgniteCheckedException If failed.
      */
+    @SuppressWarnings("unchecked")
     public UUID registerStaticRoutine(
         String cacheName,
         CacheEntryUpdatedListener<?, ?> locLsnr,
diff --git a/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java b/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java
new file mode 100644
index 0000000000000..e0688bccd934d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.custom;
+
+import javax.cache.configuration.Factory;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryEventFilter;
+import javax.cache.event.CacheEntryListenerException;
+
+/**
+ * Must be not in org.apache.ignite.internal
+ */
+public class DummyEventFilterFactory implements Factory<CacheEntryEventFilter<Integer, String>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override public CacheEntryEventFilter<Integer, String> create() {
+        return new DummyEventFilter();
+    }
+
+    /**
+     *
+     */
+    private static class DummyEventFilter implements CacheEntryEventFilter<Integer, String> {
+        /** {@inheritDoc} */
+        @Override public boolean evaluate(
+            final CacheEntryEvent<? extends Integer, ? extends String> evt) throws CacheEntryListenerException {
+            return true;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java
new file mode 100644
index 0000000000000..e5d1d609d7052
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryListenerException;
+import javax.cache.event.CacheEntryUpdatedListener;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.custom.DummyEventFilterFactory;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Checks if filter factory correctly deployed on all nodes.
+ */
+public class ContinuousQueryPeerClassLoadingTest extends GridCommonAbstractTest {
+    /** */
+    public static final String CACHE_NAME = "test-cache";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setPeerClassLoadingEnabled(true);
+        cfg.setClientMode(gridName.contains("client"));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRemoteFilterFactoryClient() throws Exception {
+        check("server", "client1", "client2");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRemoteFilterFactoryServer1() throws Exception {
+        check("server1", "server2", "client");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRemoteFilterFactoryServer2() throws Exception {
+        check("server1", "server2", "server3");
+    }
+
+    /**
+     * @param node1Name Node 1 name.
+     * @param node2Name Node 2 name.
+     * @param node3Name Node 3 name.
+     */
+    private void check(String node1Name, String node2Name, String node3Name) throws Exception {
+        final Ignite node1 = startGrid(node1Name);
+
+        final IgniteCache<Integer, String> cache = node1.getOrCreateCache(CACHE_NAME);
+
+        for (int i = 0; i < 10; i++)
+            cache.put(i, String.valueOf(i));
+
+        final Ignite node2 = startGrid(node2Name);
+
+        final ContinuousQuery<Integer, String> qry1 = new ContinuousQuery<>();
+        final ContinuousQuery<Integer, String> qry2 = new ContinuousQuery<>();
+
+        qry1.setRemoteFilterFactory(new DummyEventFilterFactory());
+        qry2.setRemoteFilterFactory(new DummyEventFilterFactory());
+
+        final AtomicInteger client1Evts = new AtomicInteger(0);
+        final AtomicInteger client2Evts = new AtomicInteger(0);
+
+        final CountDownLatch latch1 = new CountDownLatch(20);
+        final CountDownLatch latch2 = new CountDownLatch(10);
+
+        qry1.setLocalListener(new CacheEntryUpdatedListener<Integer, String>() {
+            @Override public void onUpdated(
+                final Iterable<CacheEntryEvent<? extends Integer, ? extends String>> evts) throws CacheEntryListenerException {
+                System.out.println(">> Client 1 events " + evts);
+                for (CacheEntryEvent<? extends Integer, ? extends String> evt : evts)
+                    latch1.countDown();
+            }
+        });
+
+        qry2.setLocalListener(new CacheEntryUpdatedListener<Integer, String>() {
+            @Override public void onUpdated(
+                final Iterable<CacheEntryEvent<? extends Integer, ? extends String>> evts) throws CacheEntryListenerException {
+                System.out.println(">> Client 2 events " + evts);
+                for (CacheEntryEvent<? extends Integer, ? extends String> evt : evts)
+                    latch2.countDown();
+            }
+        });
+
+        final IgniteCache<Integer, String> cache1 = node2.cache(CACHE_NAME);
+
+        cache1.query(qry1);
+
+        for (int i = 10; i < 20; i++)
+            cache.put(i, String.valueOf(i));
+
+        // Fail on start second client.
+        final Ignite node3 = startGrid(node3Name);
+
+        final IgniteCache<Integer, String> cache2 = node3.cache(CACHE_NAME);
+
+        cache2.query(qry2);
+
+        for (int i = 20; i < 30; i++)
+            cache.put(i, String.valueOf(i));
+
+        assert latch1.await(5, TimeUnit.SECONDS) : latch1.getCount();
+        assert latch2.await(5, TimeUnit.SECONDS) : latch2.getCount();
+    }
+
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
index a865788cbdfc4..6b2fea0a5ab55 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationStoreEnabledTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationSwapEnabledTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryPeerClassLoadingTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryRemoteFilterMissingInClassPathSelfTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicNearEnabledSelfTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicOffheapTieredTest;
@@ -123,6 +124,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheKeepBinaryIterationNearEnabledTest.class);
         suite.addTestSuite(IgniteCacheContinuousQueryBackupQueueTest.class);
         suite.addTestSuite(IgniteCacheContinuousQueryNoUnsubscribeTest.class);
+        suite.addTestSuite(ContinuousQueryPeerClassLoadingTest.class);
 
         return suite;
     }

From 45a4cfe7ced27aa9d726b36326bfae8b006e84da Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 30 Mar 2017 01:28:20 +0300
Subject: [PATCH 083/516] IGNITE-4141 - JDBC driver should always set
 withKeepBinary flag when querying cache. This fixes #1617.

(Backport from master)
(cherry picked from commit 8dd88d8)
---
 .../jdbc/AbstractJdbcPojoQuerySelfTest.java   | 169 ++++++++++++++++++
 .../jdbc/JdbcPojoLegacyQuerySelfTest.java     |  44 +++++
 .../ignite/jdbc/JdbcPojoQuerySelfTest.java    |  56 ++++++
 .../jdbc/suite/IgniteJdbcDriverTestSuite.java |   4 +
 .../ignite/internal/jdbc2/JdbcQueryTask.java  |   2 +-
 .../internal/jdbc2/JdbcQueryTaskV2.java       |   2 +-
 .../query/jdbc/GridCacheQueryJdbcTask.java    |   4 +-
 7 files changed, 277 insertions(+), 4 deletions(-)
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/AbstractJdbcPojoQuerySelfTest.java
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoLegacyQuerySelfTest.java
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoQuerySelfTest.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/AbstractJdbcPojoQuerySelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/AbstractJdbcPojoQuerySelfTest.java
new file mode 100644
index 0000000000000..a150574893cb2
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/AbstractJdbcPojoQuerySelfTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc;
+
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Collections;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.config.GridTestProperties;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * Test for Jdbc driver query without class on client
+ */
+public abstract class AbstractJdbcPojoQuerySelfTest extends GridCommonAbstractTest {
+    /** IP finder. */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** TestObject class name. */
+    protected static final String TEST_OBJECT = "org.apache.ignite.internal.JdbcTestObject";
+
+    /** TestObject class name. */
+    protected static final String TEST_OBJECT_2 = "org.apache.ignite.internal.JdbcTestObject2";
+
+    /** Statement. */
+    protected Statement stmt;
+
+    /** */
+    private String marshallerBackup = GridTestProperties.getProperty(GridTestProperties.MARSH_CLASS_NAME);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, BinaryMarshaller.class.getName());
+
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration<?,?> cache = defaultCacheConfiguration();
+
+        cache.setWriteSynchronizationMode(FULL_SYNC);
+        cache.setAtomicityMode(TRANSACTIONAL);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(disco);
+        cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+        QueryEntity queryEntity = new QueryEntity();
+        queryEntity.setKeyType("java.lang.String");
+        queryEntity.setValueType("org.apache.ignite.internal.JdbcTestObject");
+        queryEntity.addQueryField("id", "java.lang.Integer", null);
+        queryEntity.addQueryField("testObject", "org.apache.ignite.internal.JdbcTestObject2", null);
+        queryEntity.setIndexes(Collections.singletonList(new QueryIndex("id")));
+
+        cache.setQueryEntities(Collections.singletonList(queryEntity));
+
+        cfg.setCacheConfiguration(cache);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        Ignite ignite = startGrid(0);
+
+        BinaryObjectBuilder builder = ignite.binary().builder(TEST_OBJECT);
+        BinaryObjectBuilder builder2 = ignite.binary().builder(TEST_OBJECT_2);
+
+        builder2.setField("id", 1);
+        builder2.setField("boolVal", true);
+
+        BinaryObject testObject = builder2.build();
+
+        builder.setField("id", 1);
+        builder.setField("testObject", testObject);
+
+        BinaryObject binObj = builder.build();
+
+        IgniteCache<String, BinaryObject> cache = grid(0).cache(null);
+
+        cache.put("0", binObj);
+
+        Class.forName("org.apache.ignite.IgniteJdbcDriver");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, marshallerBackup);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stmt = DriverManager.getConnection(getURL()).createStatement();
+
+        assertNotNull(stmt);
+        assertFalse(stmt.isClosed());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        if (stmt != null) {
+            stmt.getConnection().close();
+            stmt.close();
+
+            assertTrue(stmt.isClosed());
+        }
+    }
+
+    /**
+     * @param rs Result set.
+     * @throws Exception In case of error.
+     */
+    protected void assertResultSet(ResultSet rs) throws Exception {
+        assertNotNull(rs);
+
+        int cnt = 0;
+
+        while (rs.next()) {
+            assertNotNull(rs.getString("id"));
+            assertNotNull(rs.getString("testObject"));
+
+            assertTrue(rs.getObject("testObject").toString().contains("id=1"));
+            assertTrue(rs.getObject("testObject").toString().contains("boolVal=true"));
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+    }
+
+    /**
+     * @return URL.
+     */
+    protected abstract String getURL();
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoLegacyQuerySelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoLegacyQuerySelfTest.java
new file mode 100644
index 0000000000000..4fa7ba5443870
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoLegacyQuerySelfTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc;
+
+import java.sql.ResultSet;
+
+/**
+ * Test for Jdbc driver query without class on client
+ */
+public class JdbcPojoLegacyQuerySelfTest extends AbstractJdbcPojoQuerySelfTest {
+    /** URL. */
+    private static final String URL = "jdbc:ignite://127.0.0.1/";
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testJdbcQuery() throws Exception {
+        stmt.execute("select * from JdbcTestObject");
+
+        ResultSet rs = stmt.getResultSet();
+
+        assertResultSet(rs);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected String getURL() {
+        return URL;
+    }
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoQuerySelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoQuerySelfTest.java
new file mode 100644
index 0000000000000..c2af8a1a5583f
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPojoQuerySelfTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc;
+
+import java.sql.ResultSet;
+
+import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX;
+
+/**
+ * Test for Jdbc driver query without class on client
+ */
+public class JdbcPojoQuerySelfTest extends AbstractJdbcPojoQuerySelfTest {
+    /** URL. */
+    private static final String URL = CFG_URL_PREFIX + "modules/clients/src/test/config/jdbc-bin-config.xml";
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testJdbcQueryTask2() throws Exception {
+        stmt.execute("select * from JdbcTestObject");
+
+        ResultSet rs = stmt.getResultSet();
+
+        assertResultSet(rs);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testJdbcQueryTask1() throws Exception {
+        ResultSet rs = stmt.executeQuery("select * from JdbcTestObject");
+
+        assertResultSet(rs);
+
+    }
+
+    /** {@inheritDoc} */
+    @Override protected String getURL() {
+        return URL;
+    }
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index 048643b75dd04..a28d29e00c52b 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -25,6 +25,8 @@
 import org.apache.ignite.jdbc.JdbcLocalCachesSelfTest;
 import org.apache.ignite.jdbc.JdbcMetadataSelfTest;
 import org.apache.ignite.jdbc.JdbcNoDefaultCacheTest;
+import org.apache.ignite.jdbc.JdbcPojoLegacyQuerySelfTest;
+import org.apache.ignite.jdbc.JdbcPojoQuerySelfTest;
 import org.apache.ignite.jdbc.JdbcPreparedStatementSelfTest;
 import org.apache.ignite.jdbc.JdbcResultSetSelfTest;
 import org.apache.ignite.jdbc.JdbcStatementSelfTest;
@@ -50,6 +52,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(JdbcEmptyCacheSelfTest.class));
         suite.addTest(new TestSuite(JdbcLocalCachesSelfTest.class));
         suite.addTest(new TestSuite(JdbcNoDefaultCacheTest.class));
+        suite.addTest(new TestSuite(JdbcPojoQuerySelfTest.class));
+        suite.addTest(new TestSuite(JdbcPojoLegacyQuerySelfTest.class));
 
         // Ignite client node based driver tests
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcConnectionSelfTest.class));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java
index 0b23f9b18d0e7..bd6b0f2c74c98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java
@@ -158,7 +158,7 @@ public JdbcQueryTask(Ignite ignite, String cacheName, String sql,
             qry.setCollocated(collocatedQry);
             qry.setDistributedJoins(distributedJoins);
 
-            QueryCursor<List<?>> qryCursor = cache.query(qry);
+            QueryCursor<List<?>> qryCursor = cache.withKeepBinary().query(qry);
 
             Collection<GridQueryFieldMetadata> meta = ((QueryCursorImpl<List<?>>)qryCursor).fieldsMeta();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java
index 9093d1500e65a..61f152dd66000 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java
@@ -161,7 +161,7 @@ public JdbcQueryTaskV2(Ignite ignite, String cacheName, String sql,
             qry.setCollocated(collocatedQry);
             qry.setDistributedJoins(distributedJoins);
 
-            QueryCursorImpl<List<?>> qryCursor = (QueryCursorImpl<List<?>>)cache.query(qry);
+            QueryCursorImpl<List<?>> qryCursor = (QueryCursorImpl<List<?>>)cache.withKeepBinary().query(qry);
 
             if (isQry == null)
                 isQry = qryCursor.isQuery();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/jdbc/GridCacheQueryJdbcTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/jdbc/GridCacheQueryJdbcTask.java
index ca08ead4f11eb..4ae8a8a92fdf5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/jdbc/GridCacheQueryJdbcTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/jdbc/GridCacheQueryJdbcTask.java
@@ -268,7 +268,7 @@ private static class JdbcDriverJob extends ComputeJobAdapter {
 
                 qry.setPageSize(pageSize);
 
-                QueryCursor<List<?>> cursor = cache.query(qry);
+                QueryCursor<List<?>> cursor = cache.withKeepBinary().query(qry);
 
                 Collection<GridQueryFieldMetadata> meta = ((QueryCursorImpl<List<?>>)cursor).fieldsMeta();
 
@@ -453,4 +453,4 @@ private Cursor(QueryCursor<List<?>> cursor, Iterator<List<?>> iter, int totalCnt
             return iter;
         }
     }
-}
\ No newline at end of file
+}

From d4979b87e3301f11f52531645028ecd4e947986b Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 30 Mar 2017 17:35:28 +0300
Subject: [PATCH 084/516] IGNITE-4284 - Fix. Failed second client node join
 with continuous query and peer class loading enabled.

---
 .../continuous/GridContinuousProcessor.java   | 35 ++++++++-----------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 6a4f57d1e3ae8..391c266079d2a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -503,28 +503,23 @@ public void unlockStopping() {
                         UUID routineId = e.getKey();
                         LocalRoutineInfo info = e.getValue();
 
-                        GridCacheContext cctx = ctx.cache().context().cacheContext(CU.cacheId(info.hnd.cacheName()));
-
-                        // Do not register handler if it's not affinity node.
-                        if (cctx == null || cctx.affinityNode()) {
-                            try {
-                                if (info.prjPred != null)
-                                    ctx.resource().injectGeneric(info.prjPred);
-
-                                if (info.prjPred == null || info.prjPred.apply(ctx.discovery().localNode())) {
-                                    registerHandler(clientNodeId,
-                                        routineId,
-                                        info.hnd,
-                                        info.bufSize,
-                                        info.interval,
-                                        info.autoUnsubscribe,
-                                        false);
-                                }
-                            }
-                            catch (IgniteCheckedException err) {
-                                U.error(log, "Failed to register continuous handler.", err);
+                        try {
+                            if (info.prjPred != null)
+                                ctx.resource().injectGeneric(info.prjPred);
+
+                            if (info.prjPred == null || info.prjPred.apply(ctx.discovery().localNode())) {
+                                registerHandler(clientNodeId,
+                                    routineId,
+                                    info.hnd,
+                                    info.bufSize,
+                                    info.interval,
+                                    info.autoUnsubscribe,
+                                    false);
                             }
                         }
+                        catch (IgniteCheckedException err) {
+                            U.error(log, "Failed to register continuous handler.", err);
+                        }
                     }
                 }
 

From 2e6bc4416a1f2fa18dad78235b3f71f6fb58a98d Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 30 Mar 2017 17:36:22 +0300
Subject: [PATCH 085/516] Cleanup unused imports.

---
 .../internal/processors/continuous/GridContinuousProcessor.java  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 391c266079d2a..4717e8724d038 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -67,7 +67,6 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;

From e01aee0b1bbdb8ff5583728e539df165029f682d Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 31 Mar 2017 20:19:52 +0300
Subject: [PATCH 086/516] Fixed SSL issue.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../tcp/TcpCommunicationSpi.java              | 35 +++++++++++++++++--
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 791a7d52dbd0b..89ecc36d2ace1 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3194,7 +3194,7 @@ else if (log.isDebugEnabled())
 
                             buf = ByteBuffer.allocate(1000);
 
-                            ByteBuffer decode = null;
+                            ByteBuffer decode = ByteBuffer.allocate(2 * buf.capacity());
 
                             buf.order(ByteOrder.nativeOrder());
 
@@ -3207,13 +3207,17 @@ else if (log.isDebugEnabled())
 
                                 buf.flip();
 
-                                decode = sslHnd.decode(buf);
+                                ByteBuffer decode0 = sslHnd.decode(buf);
 
-                                i += decode.remaining();
+                                i += decode0.remaining();
+
+                                decode = appendAndResizeIfNeeded(decode, decode0);
 
                                 buf.clear();
                             }
 
+                            decode.flip();
+
                             rcvCnt = decode.getLong(1);
 
                             if (decode.limit() > 9) {
@@ -3301,6 +3305,31 @@ else if (log.isDebugEnabled())
                 "is node stopping?) [senderNodeId=" + sndId + ", msg=" + msg + ']');
     }
 
+    /**
+     * @param target Target buffer to append to.
+     * @param src Source buffer to get data.
+     * @return Original or expanded buffer.
+     */
+    private ByteBuffer appendAndResizeIfNeeded(ByteBuffer target, ByteBuffer src) {
+        if (target.remaining() < src.remaining()) {
+            int newSize = Math.max(target.capacity() * 2, target.capacity() + src.remaining());
+
+            ByteBuffer tmp = ByteBuffer.allocate(newSize);
+
+            tmp.order(target.order());
+
+            target.flip();
+
+            tmp.put(target);
+
+            target = tmp;
+        }
+
+        target.put(src);
+
+        return target;
+    }
+
     /**
      * Stops service threads to simulate node failure.
      *

From 9d8de41b6e8ed6ec8a29179b68a05c93d9077045 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Mon, 3 Apr 2017 14:19:56 +0700
Subject: [PATCH 087/516] IGNITE-4659 Cleanup after merge.

---
 .../activities-user-dialog.jade               | 36 --------
 .../form-field-datepicker.jade                | 55 -------------
 .../list-of-registered-users.jade             | 58 -------------
 .../ui-grid-header/ui-grid-header.jade        | 27 ------
 .../states/configuration/caches/affinity.jade | 82 -------------------
 5 files changed, 258 deletions(-)
 delete mode 100644 modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
 delete mode 100644 modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
 delete mode 100644 modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
 delete mode 100644 modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
 delete mode 100644 modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade

diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
deleted file mode 100644
index 074851ca55d66..0000000000000
--- a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.jade
+++ /dev/null
@@ -1,36 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-.modal(tabindex='-1' role='dialog')
-    .modal-dialog
-        .modal-content
-            .modal-header
-                h4.modal-title 
-                    i.fa.fa-info-circle
-                    | Activities: {{ ctrl.user.userName }}
-            .modal-body.modal-body-with-scroll(id='activities-user-dialog')
-                table.table.table-striped.table-bordered.table-hover(scrollable-container='#activities-user-dialog' st-table='displayedRows' st-safe-src='ctrl.data')
-                    thead
-                        th.text-center(st-sort='action | translate') Description
-                        th.text-center(st-sort='action') Action
-                        th.text-center(st-sort='amount') Visited
-                    tbody
-                        tr(ng-repeat='row in displayedRows')
-                            td.text-left {{ row.action | translate }}
-                            td.text-left {{ row.action }}
-                            td.text-left {{ row.amount }}
-            .modal-footer
-                button.btn.btn-primary(id='confirm-btn-confirm' ng-click='$hide()') Close
diff --git a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade b/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
deleted file mode 100644
index 2578cf4b17189..0000000000000
--- a/modules/web-console/frontend/app/components/form-field-datepicker/form-field-datepicker.jade
+++ /dev/null
@@ -1,55 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-mixin ignite-form-field-datepicker(label, model, name, disabled, required, placeholder, tip)
-    mixin form-field-input()
-        input.form-control(
-            id='{{ #{name} }}Input'
-            name='{{ #{name} }}'
-
-            placeholder=placeholder
-            
-            data-ng-model=model
-
-            data-ng-required=required && '#{required}'
-            data-ng-disabled=disabled && '#{disabled}'
-
-            bs-datepicker
-            data-date-format='MMM yyyy'
-            data-start-view='1'
-            data-min-view='1' 
-            data-max-date='today'
-
-            data-container='body > .wrapper'
-
-            tabindex='0'
-
-            onkeydown="return false"
-
-            data-ignite-form-panel-field=''
-        )&attributes(attributes.attributes)
-
-    .ignite-form-field
-        +ignite-form-field__label(label, name, required)
-        .ignite-form-field__control
-            if tip
-                i.tipField.icon-help(bs-tooltip='' data-title=tip)
-
-            if block
-                block
-
-            .input-tip
-                +form-field-input(attributes=attributes)
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
deleted file mode 100644
index 119591036d9da..0000000000000
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.jade
+++ /dev/null
@@ -1,58 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-include /app/helpers/jade/mixins.jade
-include /app/components/form-field-datepicker/form-field-datepicker.jade
-
-mixin grid-settings()
-    i.fa.fa-bars(data-animation='am-flip-x' bs-dropdown='' aria-haspopup='true' aria-expanded='expanded' data-auto-close='1' data-trigger='click')
-    ul.select.dropdown-menu(role='menu')
-        li(ng-repeat='item in $ctrl.gridOptions.categories|filter:{selectable:true}')
-            a(ng-click='$ctrl.toggleColumns(item, !item.visible)')
-                i.fa.fa-check-square-o.pull-left(ng-if='item.visible')
-                i.fa.fa-square-o.pull-left(ng-if='!item.visible')
-                span {{::item.name}}
-        li.divider
-        li
-            a(ng-click='$ctrl.selectAllColumns()') Select all
-        li
-            a(ng-click='$ctrl.clearAllColumns()') Clear all
-        li.divider
-        li
-            a(ng-click='$hide()') Close
-
-.panel.panel-default
-    .panel-heading.ui-grid-settings
-        +grid-settings
-        label Total users: 
-            strong {{ $ctrl.gridOptions.data.length }}&nbsp;&nbsp;&nbsp;
-        label Showing users:
-            strong {{ $ctrl.gridApi.grid.getVisibleRows().length }}
-            sub(ng-show='users.length === $ctrl.gridApi.grid.getVisibleRows().length') all
-
-        form.pull-right(ng-form=form novalidate)
-            -var form = 'admin'
-
-            button.btn.btn-primary(ng-click='$ctrl.exportCsv()' bs-tooltip data-title='Export table to csv') Export
-
-            .ui-grid-settings-dateperiod
-                +ignite-form-field-datepicker('Period:', '$ctrl.params.startDate', '"period"')
-
-            .ui-grid-settings-filter
-                +ignite-form-field-text('Exclude:', '$ctrl.params.companiesExclude', '"exclude"', false, false, 'Exclude by company name...')
-
-    .panel-collapse
-        .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-exporter ui-grid-pinning)
diff --git a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade b/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
deleted file mode 100644
index 7e44d94671452..0000000000000
--- a/modules/web-console/frontend/app/components/ui-grid-header/ui-grid-header.jade
+++ /dev/null
@@ -1,27 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-.ui-grid-header.ui-grid-header--subcategories(role='rowgroup')
-    .ui-grid-top-panel
-        .ui-grid-header-viewport
-            .ui-grid-header-canvas
-                .ui-grid-header-cell-wrapper(ng-style='colContainer.headerCellWrapperStyle()')
-                    .ui-grid-header-cell-row(role='row')
-                        .ui-grid-header-span.ui-grid-header-cell.ui-grid-clearfix(ng-repeat='cat in grid.options.categories')
-                            div(ng-show='(colContainer.renderedColumns|uiGridSubcategories:cat.name).length > 1')
-                                .ui-grid-cell-contents {{ cat.name }}
-                            .ui-grid-header-cell-row
-                                .ui-grid-header-cell.ui-grid-clearfix(ng-repeat='col in (colContainer.renderedColumns|uiGridSubcategories:cat.name) track by col.uid' ui-grid-header-cell='' col='col' render-index='$index')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade
deleted file mode 100644
index 3c4746bbca794..0000000000000
--- a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.jade
+++ /dev/null
@@ -1,82 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-include /app/helpers/jade/mixins.jade
-
--var form = 'affinity'
--var model = 'backupItem'
--var affModel = model + '.affinity'
--var affMapModel = model + '.affinityMapper'
--var rendezvousAff = affModel + '.kind === "Rendezvous"'
--var fairAff = affModel + '.kind === "Fair"'
--var customAff = affModel + '.kind === "Custom"'
--var customAffMapper = affMapModel + '.kind === "Custom"'
--var rendPartitionsRequired = rendezvousAff + ' && ' + affModel + '.Rendezvous.affinityBackupFilter'
--var fairPartitionsRequired = fairAff + ' && ' + affModel + '.Fair.affinityBackupFilter'
-
-.panel.panel-default(ng-form=form novalidate)
-    .panel-heading(bs-collapse-toggle='' ng-click='ui.loadPanel("#{form}")')
-        ignite-form-panel-chevron
-        label Affinity Collocation
-        ignite-form-field-tooltip.tipLabel
-            | Collocate data with data to improve performance and scalability of your application#[br]
-            | #[a(href="http://apacheignite.gridgain.org/docs/affinity-collocation" target="_blank") More info]
-        ignite-form-revert
-    .panel-collapse(role='tabpanel' bs-collapse-target id=form)
-        .panel-body(ng-if='ui.isPanelLoaded("#{form}")')
-            .col-sm-6
-                .settings-row
-                    +dropdown('Function:', affModel + '.kind', '"AffinityKind"', 'true', 'Default',
-                        '[\
-                            {value: "Rendezvous", label: "Rendezvous"},\
-                            {value: "Fair", label: "Fair"},\
-                            {value: "Custom", label: "Custom"},\
-                            {value: undefined, label: "Default"}\
-                        ]',
-                        'Key topology resolver to provide mapping from keys to nodes\
-                        <ul>\
-                            <li>Rendezvous - Based on Highest Random Weight algorithm<br/></li>\
-                            <li>Fair - Tries to ensure that all nodes get equal number of partitions with minimum amount of reassignments between existing nodes<br/></li>\
-                            <li>Custom - Custom implementation of key affinity fynction<br/></li>\
-                            <li>Default - By default rendezvous affinity function  with 1024 partitions is used<br/></li>\
-                        </ul>')
-                .panel-details(ng-if=rendezvousAff)
-                    .details-row
-                        +number-required('Partitions', affModel + '.Rendezvous.partitions', '"RendPartitions"', 'true', rendPartitionsRequired, '1024', '1', 'Number of partitions')
-                    .details-row
-                        +java-class('Backup filter', affModel + '.Rendezvous.affinityBackupFilter', '"RendAffinityBackupFilter"', 'true', 'false',
-                            'Backups will be selected from all nodes that pass this filter')
-                    .details-row
-                        +checkbox('Exclude neighbors', affModel + '.Rendezvous.excludeNeighbors', '"RendExcludeNeighbors"',
-                            'Exclude same - host - neighbors from being backups of each other and specified number of backups')
-                .panel-details(ng-if=fairAff)
-                    .details-row
-                        +number-required('Partitions', affModel + '.Fair.partitions', '"FairPartitions"', 'true', fairPartitionsRequired, '256', '1', 'Number of partitions')
-                    .details-row
-                        +java-class('Backup filter', affModel + '.Fair.affinityBackupFilter', '"FairAffinityBackupFilter"', 'true', 'false',
-                            'Backups will be selected from all nodes that pass this filter')
-                    .details-row
-                        +checkbox('Exclude neighbors', affModel + '.Fair.excludeNeighbors', '"FairExcludeNeighbors"',
-                            'Exclude same - host - neighbors from being backups of each other and specified number of backups')
-                .panel-details(ng-if=customAff)
-                    .details-row
-                        +java-class('Class name:', affModel + '.Custom.className', '"AffCustomClassName"', 'true', customAff,
-                            'Custom key affinity function implementation class name')
-                .settings-row
-                    +java-class('Mapper:', model + '.affinityMapper', '"AffMapCustomClassName"', 'true', 'false',
-                        'Provide custom affinity key for any given key')
-            .col-sm-6
-                +preview-xml-java(model, 'cacheAffinity')

From 925ee11c2002729b1264148ee5db5700ded5a7b7 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 4 Apr 2017 16:06:01 +0700
Subject: [PATCH 088/516] Fixed typo. (cherry picked from commit 3b84f34)

---
 .../app/modules/states/configuration/clusters/general.pug       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
index be56a6c44d08f..dfc49d61ef4c2 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
@@ -49,7 +49,7 @@ include /app/helpers/jade/mixins
                             <li>AWS S3 - AWS S3 based IP finder that automatically discover cluster nodes on Amazon EC2 cloud</li>\
                             <li>Apache jclouds - Apache jclouds multi cloud toolkit based IP finder for cloud platforms with unstable IP addresses</li>\
                             <li>Google cloud storage - Google Cloud Storage based IP finder that automatically discover cluster nodes on Google Compute Engine cluster</li>\
-                            <li>JDBC - JDBC based IP finder that use database to store node IP addres</li>\
+                            <li>JDBC - JDBC based IP finder that use database to store node IP address</li>\
                             <li>Shared filesystem - Shared filesystem based IP finder that use file to store node IP address</li>\
                             <li>Apache ZooKeeper - Apache ZooKeeper based IP finder when you use ZooKeeper to coordinate your distributed environment</li>\
                         </ul>')

From ce4b078c1c97cae4136c318ae38b27a50fe383cc Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 4 Apr 2017 16:14:56 +0700
Subject: [PATCH 089/516] master Updated version. (cherry picked from commit
 5469626)

---
 .../frontend/app/modules/configuration/Version.service.js       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/app/modules/configuration/Version.service.js b/modules/web-console/frontend/app/modules/configuration/Version.service.js
index 3fc719946729f..164bd20c69a0f 100644
--- a/modules/web-console/frontend/app/modules/configuration/Version.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/Version.service.js
@@ -24,7 +24,7 @@ const numberComparator = (a, b) => a > b ? 1 : a < b ? -1 : 0;
 
 export default class IgniteVersion {
     /** Current product version. */
-    static ignite = '1.8.0';
+    static ignite = '1.9.0';
 
     /**
      * Tries to parse product version from it's string representation.

From ae435fb5474429e7ac76faadd680c19a4c657371 Mon Sep 17 00:00:00 2001
From: Sergi Vladykin <sergi.vladykin@gmail.com>
Date: Sun, 2 Apr 2017 15:35:04 +0300
Subject: [PATCH 090/516] ignite-2913 - SQL: EXISTS support added

---
 .../query/h2/sql/GridSqlOperationType.java    | 23 ++++-
 .../query/h2/sql/GridSqlQueryParser.java      | 19 +++-
 .../query/IgniteSqlSplitterSelfTest.java      | 94 ++++++++++++++-----
 .../query/h2/sql/GridQueryParsingTest.java    | 91 +++++++++++++++---
 4 files changed, 183 insertions(+), 44 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java
index 8d3165148219f..9ffabad1b613f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java
@@ -31,7 +31,7 @@ public enum GridSqlOperationType {
     MULTIPLY(2, new BiExpressionSqlGenerator("*")),
     DIVIDE(2, new BiExpressionSqlGenerator("/")),
     MODULUS(2, new BiExpressionSqlGenerator("%")),
-    NEGATE(1, new PrefixSqlGenerator("-")),
+    NEGATE(1, new PrefixSqlGenerator("-", true)),
 
     // from org.h2.expression.Comparison
     EQUAL(2, new BiExpressionSqlGenerator("=")),
@@ -47,7 +47,7 @@ public enum GridSqlOperationType {
     IS_NULL(1, new SuffixSqlGenerator("IS NULL")),
     IS_NOT_NULL(1, new SuffixSqlGenerator("IS NOT NULL")),
 
-    NOT(1, new PrefixSqlGenerator("NOT")),
+    NOT(1, new PrefixSqlGenerator("NOT", true)),
 
     // from org.h2.expression.ConditionAndOr
     AND(2, new BiExpressionSqlGenerator("AND")),
@@ -58,6 +58,7 @@ public enum GridSqlOperationType {
     LIKE(2, new BiExpressionSqlGenerator("LIKE")),
 
     IN(-1, new ConditionInSqlGenerator()),
+    EXISTS(1, new PrefixSqlGenerator("EXISTS", false)),
 
     ;
     /** */
@@ -145,18 +146,32 @@ private static class PrefixSqlGenerator implements SqlGenerator {
         /** */
         private final String text;
 
+        /** */
+        private final boolean addSpace;
+
         /**
          * @param text Text.
+         * @param addSpace Add space char after the prefix.
          */
-        private PrefixSqlGenerator(String text) {
+        private PrefixSqlGenerator(String text, boolean addSpace) {
             this.text = text;
+            this.addSpace = addSpace;
         }
 
         /** {@inheritDoc} */
         @Override public String getSql(GridSqlOperation operation) {
             assert operation.operationType().childrenCnt == 1;
 
-            return '(' + text + ' ' + operation.child().getSQL() + ')';
+            StringBuilder b = new StringBuilder();
+
+            b.append('(').append(text);
+
+            if (addSpace)
+                b.append(' ');
+
+            b.append(operation.child(0).getSQL()).append(')');
+
+            return b.toString();
         }
     }
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index d9c546c7fb104..a3e80ec6424da 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -46,6 +46,7 @@
 import org.h2.expression.CompareLike;
 import org.h2.expression.Comparison;
 import org.h2.expression.ConditionAndOr;
+import org.h2.expression.ConditionExists;
 import org.h2.expression.ConditionIn;
 import org.h2.expression.ConditionInConstantSet;
 import org.h2.expression.ConditionInSelect;
@@ -80,6 +81,7 @@
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.DIVIDE;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.EQUAL;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.EQUAL_NULL_SAFE;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.EXISTS;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.IN;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.IS_NOT_NULL;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType.IS_NULL;
@@ -186,7 +188,10 @@ public class GridSqlQueryParser {
         "compareType");
 
     /** */
-    private static final Getter<ConditionInSelect, Query> QUERY = getter(ConditionInSelect.class, "query");
+    private static final Getter<ConditionInSelect, Query> QUERY_IN = getter(ConditionInSelect.class, "query");
+
+    /** */
+    private static final Getter<ConditionExists, Query> QUERY_EXISTS = getter(ConditionExists.class, "query");
 
     /** */
     private static final Getter<CompareLike, Expression> LEFT = getter(CompareLike.class, "left");
@@ -854,7 +859,7 @@ private GridSqlElement parseExpression0(Expression expression, boolean calcTypes
 
             res.addChild(parseExpression(LEFT_CIS.get((ConditionInSelect)expression), calcTypes));
 
-            Query qry = QUERY.get((ConditionInSelect)expression);
+            Query qry = QUERY_IN.get((ConditionInSelect)expression);
 
             assert0(qry instanceof Select, qry);
 
@@ -959,6 +964,16 @@ private GridSqlElement parseExpression0(Expression expression, boolean calcTypes
             return res;
         }
 
+        if (expression instanceof ConditionExists) {
+            Query qry = QUERY_EXISTS.get((ConditionExists)expression);
+
+            GridSqlOperation res = new GridSqlOperation(EXISTS);
+
+            res.addChild(new GridSqlSubquery(parse(qry, null)));
+
+            return res;
+        }
+
         throw new IgniteException("Unsupported expression: " + expression + " [type=" +
             expression.getClass().getSimpleName() + ']');
     }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index e72c9cb5b6d62..c3f1bd277a2de 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -21,9 +21,11 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicLong;
 import javax.cache.CacheException;
@@ -33,10 +35,8 @@
 import org.apache.ignite.cache.CacheKeyConfiguration;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CachePeekMode;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
@@ -63,8 +63,8 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
     private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
         CacheKeyConfiguration keyCfg = new CacheKeyConfiguration(TestKey.class.getName(), "affKey");
 
@@ -81,6 +81,11 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    @Override
+    protected long getTestTimeout() {
+        return 100_000_000;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         startGridsMultiThreaded(3, false);
@@ -148,6 +153,47 @@ public void testOffsetLimit() throws Exception {
         }
     }
 
+    @SuppressWarnings("SuspiciousMethodCalls")
+    public void testExists() {
+        IgniteCache<Integer,Person2> x = ignite(0).getOrCreateCache(cacheConfig("x", true,
+            Integer.class, Person2.class));
+        IgniteCache<Integer,Person2> y = ignite(0).getOrCreateCache(cacheConfig("y", true,
+            Integer.class, Person2.class));
+
+        try {
+            GridRandom rnd = new GridRandom();
+
+            Set<Integer> intersects = new HashSet<>();
+
+            for (int i = 0; i < 3000; i++) {
+                int r = rnd.nextInt(3);
+
+                if (r != 0)
+                    x.put(i, new Person2(i, "pers_x_" + i));
+
+                if (r != 1)
+                    y.put(i, new Person2(i, "pers_y_" + i));
+
+                if (r == 2)
+                    intersects.add(i);
+            }
+
+            assertFalse(intersects.isEmpty());
+
+            List<List<?>> res = x.query(new SqlFieldsQuery("select _key from \"x\".Person2 px " +
+                "where exists(select 1 from \"y\".Person2 py where px._key = py._key)")).getAll();
+
+            assertEquals(intersects.size(), res.size());
+
+            for (List<?> row : res)
+                assertTrue(intersects.contains(row.get(0)));
+        }
+        finally {
+            x.destroy();
+            y.destroy();
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -550,15 +596,15 @@ public void testDistributedJoinsPlan() throws Exception {
                 "\"orgRepl\".Organization o",
                 "where p.affKey = o._key", true);
 
-            checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
-                "(select * from \"persPart\".Person2) p",
-                "\"orgPart\".Organization o",
-                "where p._key = o._key", false);
-
-            checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
-                "\"persPart\".Person2 p",
-                "(select * from \"orgPart\".Organization) o",
-                "where p._key = o._key", false);
+            // TODO Now we can not analyze subqueries to decide if we are collocated or not.
+//            checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
+//                "(select * from \"persPart\".Person2) p",
+//                "\"orgPart\".Organization o",
+//                "where p._key = o._key", false);
+//            checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
+//                "\"persPart\".Person2 p",
+//                "(select * from \"orgPart\".Organization) o",
+//                "where p._key = o._key", false);
 
             // Join multiple.
 
@@ -703,6 +749,7 @@ public void testDistributedJoinsPlan() throws Exception {
                 ignite(0).destroyCache(cache.getName());
         }
     }
+
     /**
      * @throws Exception If failed.
      */
@@ -791,26 +838,26 @@ private void checkNoBatchedJoin(IgniteCache<Object, Object> cache,
             false,
             0,
             select +
-                "from " + cache1 + ","  + cache2 + " "+ where);
+                "from " + cache1 + "," + cache2 + " " + where);
 
         checkQueryPlan(cache,
             false,
             0,
             select +
-                "from " + cache2 + ","  + cache1 + " "+ where);
+                "from " + cache2 + "," + cache1 + " " + where);
 
         if (testEnforceJoinOrder) {
             checkQueryPlan(cache,
                 true,
                 0,
                 select +
-                    "from " + cache1 + ","  + cache2 + " "+ where);
+                    "from " + cache1 + "," + cache2 + " " + where);
 
             checkQueryPlan(cache,
                 true,
                 0,
                 select +
-                    "from " + cache2 + ","  + cache1 + " "+ where);
+                    "from " + cache2 + "," + cache1 + " " + where);
         }
     }
 
@@ -825,7 +872,8 @@ private void checkQueryPlan(IgniteCache<Object, Object> cache,
         boolean enforceJoinOrder,
         int expBatchedJoins,
         String sql,
-        String...expText) {
+        String...expText
+    ) {
         checkQueryPlan(cache,
             enforceJoinOrder,
             expBatchedJoins,
@@ -850,13 +898,13 @@ private void checkQueryPlan(IgniteCache<Object, Object> cache,
         boolean enforceJoinOrder,
         int expBatchedJoins,
         SqlFieldsQuery qry,
-        String...expText) {
+        String... expText) {
         qry.setEnforceJoinOrder(enforceJoinOrder);
         qry.setDistributedJoins(true);
 
         String plan = queryPlan(cache, qry);
 
-        log.info("Plan: " + plan);
+        log.info("\n  Plan:\n" + plan);
 
         assertEquals("Unexpected number of batched joins in plan [plan=" + plan + ", qry=" + qry + ']',
             expBatchedJoins,
@@ -986,7 +1034,7 @@ private void doTestDistributedJoins(IgniteCache<Integer, Object> c, int orgs, in
      * @param args Arguments.
      * @return Column as list.
      */
-    private static <X> List<X> columnQuery(IgniteCache<?,?> c, String qry, Object... args) {
+    private static <X> List<X> columnQuery(IgniteCache<?, ?> c, String qry, Object... args) {
         return column(0, c.query(new SqlFieldsQuery(qry).setArgs(args)).getAll());
     }
 
@@ -1584,4 +1632,4 @@ private static class OrderGood implements Serializable {
         @QuerySqlField
         private int goodId;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
index 537ccdfd57953..ecdb593758f8a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
@@ -39,12 +39,8 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.h2.command.Prepared;
-import org.h2.command.dml.Query;
-import org.h2.command.dml.Update;
 import org.h2.engine.Session;
-import org.h2.expression.Expression;
 import org.h2.jdbc.JdbcConnection;
-import org.h2.util.StringUtils;
 
 import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
@@ -56,13 +52,19 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
     /** */
     private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
 
+    /** */
+    private static final String TEST_SCHEMA = "SCH";
+
+    /** */
+    private static final String TEST_CACHE = "my-cache";
+
     /** */
     private static Ignite ignite;
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration c = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
 
         TcpDiscoverySpi disco = new TcpDiscoverySpi();
 
@@ -73,12 +75,14 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
         // Cache.
         CacheConfiguration cc = defaultCacheConfiguration();
 
+        cc.setName(TEST_CACHE);
         cc.setCacheMode(CacheMode.PARTITIONED);
         cc.setAtomicityMode(CacheAtomicityMode.ATOMIC);
         cc.setNearConfiguration(null);
         cc.setWriteSynchronizationMode(FULL_SYNC);
         cc.setRebalanceMode(SYNC);
         cc.setSwapEnabled(false);
+        cc.setSqlSchema(TEST_SCHEMA);
         cc.setSqlFunctionClasses(GridQueryParsingTest.class);
         cc.setIndexedTypes(
             String.class, Address.class,
@@ -110,6 +114,12 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testParseSelectAndUnion() throws Exception {
+        checkQuery("select 1 from Person p where addrIds in ((1,2,3), (3,4,5))");
+        checkQuery("select 1 from Person p where addrId in ((1,))");
+        checkQuery("select 1 from Person p " +
+            "where p.addrId in (select a.id from Address a)");
+        checkQuery("select 1 from Person p " +
+            "where exists(select 1 from Address a where p.addrId = a.id)");
         checkQuery("select 42");
         checkQuery("select ()");
         checkQuery("select (1)");
@@ -244,16 +254,16 @@ public void testParseSelectAndUnion() throws Exception {
         checkQuery("select street from Person p, (select a.street from Address a where a.street is not null) ");
         checkQuery("select addr.street from Person p, (select a.street from Address a where a.street is not null) addr");
 
-        checkQuery("select p.name n from \"\".Person p order by p.old + 10");
+        checkQuery("select p.name n from sch.Person p order by p.old + 10");
 
-        checkQuery("select case when p.name is null then 'Vasya' end x from \"\".Person p");
-        checkQuery("select case when p.name like 'V%' then 'Vasya' else 'Other' end x from \"\".Person p");
-        checkQuery("select case when upper(p.name) = 'VASYA' then 'Vasya' when p.name is not null then p.name else 'Other' end x from \"\".Person p");
+        checkQuery("select case when p.name is null then 'Vasya' end x from sch.Person p");
+        checkQuery("select case when p.name like 'V%' then 'Vasya' else 'Other' end x from sch.Person p");
+        checkQuery("select case when upper(p.name) = 'VASYA' then 'Vasya' when p.name is not null then p.name else 'Other' end x from sch.Person p");
 
-        checkQuery("select case p.name when 'Vasya' then 1 end z from \"\".Person p");
-        checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 end z from \"\".Person p");
-        checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 else 3 end z from \"\".Person p");
-        checkQuery("select case p.name when 'Vasya' then 1 else 3 end z from \"\".Person p");
+        checkQuery("select case p.name when 'Vasya' then 1 end z from sch.Person p");
+        checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 end z from sch.Person p");
+        checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 else 3 end z from sch.Person p");
+        checkQuery("select case p.name when 'Vasya' then 1 else 3 end z from sch.Person p");
 
         checkQuery("select count(*) as a from Person union select count(*) as a from Address");
         checkQuery("select old, count(*) as a from Person group by old union select 1, count(*) as a from Address");
@@ -268,6 +278,54 @@ public void testParseSelectAndUnion() throws Exception {
         checkQuery("(select 2 a) union all (select 1) order by a desc nulls first limit ? offset ?");
     }
 
+    /**
+     * Query AST transformation heavily depends on this behavior.
+     *
+     * @throws Exception If failed.
+     */
+    public void testParseTableFilter() throws Exception {
+        Prepared prepared = parse("select Person.old, p1.old, p1.addrId from Person, Person p1 " +
+            "where exists(select 1 from Address a where a.id = p1.addrId)");
+
+        GridSqlSelect select = (GridSqlSelect)new GridSqlQueryParser().parse(prepared);
+
+        GridSqlJoin join = (GridSqlJoin)select.from();
+
+        GridSqlTable tbl1 = (GridSqlTable)join.leftTable();
+        GridSqlAlias tbl2Alias = (GridSqlAlias)join.rightTable();
+        GridSqlTable tbl2 = tbl2Alias.child();
+
+        // Must be distinct objects, even if it is the same table.
+        //assertNotSame(tbl1, tbl2);
+
+        assertNotNull(tbl1.dataTable());
+        assertNotNull(tbl2.dataTable());
+        assertSame(tbl1.dataTable(), tbl2.dataTable());
+
+        GridSqlColumn col1 = (GridSqlColumn)select.column(0);
+        GridSqlColumn col2 = (GridSqlColumn)select.column(1);
+
+        assertSame(tbl1, col1.expressionInFrom());
+
+        // Alias in FROM must be included in column.
+        assertSame(tbl2Alias, col2.expressionInFrom());
+
+        // In EXISTS we must correctly reference the column from the outer query.
+        GridSqlElement exists = select.where();
+        GridSqlSubquery subqry = exists.child();
+        GridSqlSelect subSelect = (GridSqlSelect)subqry.select();
+
+        GridSqlColumn p1AddrIdCol = (GridSqlColumn)select.column(2);
+
+        assertEquals("ADDRID", p1AddrIdCol.column().getName());
+        assertSame(tbl2Alias, p1AddrIdCol.expressionInFrom());
+
+        GridSqlColumn p1AddrIdColExists = subSelect.where().child(1);
+        assertEquals("ADDRID", p1AddrIdCol.column().getName());
+
+        assertSame(tbl2Alias, p1AddrIdColExists.expressionInFrom());
+    }
+
     /** */
     public void testParseMerge() throws Exception {
         /* Plain rows w/functions, operators, defaults, and placeholders. */
@@ -378,7 +436,7 @@ private JdbcConnection connection() throws Exception {
 
         IgniteH2Indexing idx = U.field(qryProcessor, "idx");
 
-        return (JdbcConnection)idx.connectionForSpace(null);
+        return (JdbcConnection)idx.connectionForSpace(TEST_CACHE);
     }
 
     /**
@@ -455,6 +513,9 @@ public static class Person implements Serializable {
         @QuerySqlField(index = true)
         public int addrId;
 
+        @QuerySqlField
+        public Integer[] addrIds;
+
         @QuerySqlField(index = true)
         public int old;
     }

From b7ab27301b59bf93fc73b52fdf8e0bcf124fec1d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 6 Apr 2017 14:43:50 +0300
Subject: [PATCH 091/516] IGNITE-4832: Prevent service deployment on client by
 default when configuration is provided on startup. This closes #1748.

---
 .../service/GridServiceProcessor.java         |  10 +-
 .../GridServiceProcessorAbstractSelfTest.java |  11 ++
 ...rviceProcessorMultiNodeConfigSelfTest.java |  74 +++++++++-
 ...GridServiceProcessorMultiNodeSelfTest.java | 139 +++++++++++++++---
 4 files changed, 202 insertions(+), 32 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index bd815189fb5ba..a8af9832e9355 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -70,11 +70,11 @@
 import org.apache.ignite.internal.processors.cache.query.CacheQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
-import org.apache.ignite.internal.processors.continuous.AbstractContinuousMessage;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridEmptyIterator;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.SerializableTransient;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -92,7 +92,6 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
-import org.apache.ignite.internal.util.SerializableTransient;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
@@ -296,8 +295,13 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         if (cfgs != null) {
             Collection<IgniteInternalFuture<?>> futs = new ArrayList<>();
 
-            for (ServiceConfiguration c : ctx.config().getServiceConfiguration())
+            for (ServiceConfiguration c : cfgs) {
+                // Deploy only on server nodes by default.
+                if (c.getNodeFilter() == null)
+                    c.setNodeFilter(ctx.cluster().get().forServers().predicate());
+
                 futs.add(deploy(c));
+            }
 
             // Await for services to deploy.
             for (IgniteInternalFuture<?> f : futs)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
index 111cb714e8f32..0f79855518e3d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
@@ -129,6 +129,17 @@ protected void startExtraNodes(int cnt) throws Exception {
             startGrid(nodeCount() + i);
     }
 
+    /** */
+    protected void startExtraNodes(int servers, int clients) throws Exception {
+        startExtraNodes(servers);
+
+        for (int i = 0; i < clients; i++) {
+            final String nodeName = getTestGridName(nodeCount() + servers + i);
+
+            startGrid(nodeName, getConfiguration(nodeName).setClientMode(true));
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index b819cc93e6fb8..1bd3b035a446c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -19,7 +19,9 @@
 
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
+import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -33,6 +35,9 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
     /** Node singleton name. */
     private static final String NODE_SINGLE = "serviceConfigEachNode";
 
+    /** Node singleton name. */
+    private static final String NODE_SINGLE_BUT_CLIENT = "serviceConfigEachNodeButClient";
+
     /** Affinity service name. */
     private static final String AFFINITY = "serviceConfigAffinity";
 
@@ -46,7 +51,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        ServiceConfiguration[] arr = new ServiceConfiguration[3];
+        ServiceConfiguration[] arr = new ServiceConfiguration[4];
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -59,7 +64,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
         cfg = new ServiceConfiguration();
 
-        cfg.setName(NODE_SINGLE);
+        cfg.setName(NODE_SINGLE_BUT_CLIENT);
         cfg.setMaxPerNodeCount(1);
         cfg.setService(new DummyService());
 
@@ -76,6 +81,15 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
         arr[2] = cfg;
 
+        cfg = new ServiceConfiguration();
+
+        cfg.setName(NODE_SINGLE);
+        cfg.setMaxPerNodeCount(1);
+        cfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
+        cfg.setService(new DummyService());
+
+        arr[3] = cfg;
+
         return arr;
     }
 
@@ -91,6 +105,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
                         DummyService.cancelled(CLUSTER_SINGLE) == 0 &&
                         DummyService.started(NODE_SINGLE) == nodeCount() &&
                         DummyService.cancelled(NODE_SINGLE) == 0 &&
+                        DummyService.started(NODE_SINGLE_BUT_CLIENT) == nodeCount() &&
+                        DummyService.cancelled(NODE_SINGLE_BUT_CLIENT) == 0 &&
                         actualCount(AFFINITY, randomGrid().services().serviceDescriptors()) == 1;
                 }
             },
@@ -112,6 +128,13 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
         checkDeployOnEachNodeUpdateTopology(NODE_SINGLE);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
+        checkDeployOnEachNodeButClientUpdateTopology(NODE_SINGLE_BUT_CLIENT);
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -120,6 +143,10 @@ public void testAll() throws Exception {
 
         DummyService.reset();
 
+        checkDeployOnEachNodeButClientUpdateTopology(NODE_SINGLE_BUT_CLIENT);
+
+        DummyService.reset();
+
         checkDeployOnEachNodeUpdateTopology(NODE_SINGLE);
 
         DummyService.reset();
@@ -152,9 +179,7 @@ public void testAffinityUpdateTopology() throws Exception {
     private void checkSingletonUpdateTopology(String name) throws Exception {
         Ignite g = randomGrid();
 
-        int nodeCnt = 2;
-
-        startExtraNodes(nodeCnt);
+        startExtraNodes(2, 2);
 
         try {
             assertEquals(name, 0, DummyService.started(name));
@@ -165,7 +190,7 @@ private void checkSingletonUpdateTopology(String name) throws Exception {
             checkCount(name, g.services().serviceDescriptors(), 1);
         }
         finally {
-            stopExtraNodes(nodeCnt);
+            stopExtraNodes(4);
         }
     }
 
@@ -176,17 +201,21 @@ private void checkSingletonUpdateTopology(String name) throws Exception {
     private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         Ignite g = randomGrid();
 
-        int newNodes = 2;
+        int newNodes = 4;
 
         CountDownLatch latch = new CountDownLatch(newNodes);
 
         DummyService.exeLatch(name, latch);
 
-        startExtraNodes(newNodes);
+        startExtraNodes(2, 2);
 
         try {
             latch.await();
 
+            // Ensure service is deployed.
+            assertNotNull(grid(nodeCount() + newNodes - 1).services()
+                .serviceProxy(NODE_SINGLE_BUT_CLIENT, Service.class, false, 2000));
+
             assertEquals(name, newNodes, DummyService.started(name));
             assertEquals(name, 0, DummyService.cancelled(name));
 
@@ -196,4 +225,33 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
             stopExtraNodes(newNodes);
         }
     }
+
+    /**
+     * @param name Name.
+     * @throws Exception If failed.
+     */
+    private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Exception {
+        Ignite g = randomGrid();
+
+        int servers = 2;
+        int clients = 2;
+
+        CountDownLatch latch = new CountDownLatch(servers);
+
+        DummyService.exeLatch(name, latch);
+
+        startExtraNodes(servers, clients);
+
+        try {
+            latch.await();
+
+            assertEquals(name, servers, DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
+        }
+        finally {
+            stopExtraNodes(servers + clients);
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index 39336ef001a02..f7403dcc820ca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -21,7 +21,10 @@
 import junit.framework.TestCase;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteServices;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
 
 /**
  * Single node services test.
@@ -121,50 +124,144 @@ public void testAffinityDeployUpdateTopology() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testDeployOnEachNodeUpdateTopology() throws Exception {
-        String name = "serviceOnEachNodeUpdateTopology";
+    public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
-        Ignite g = randomGrid();
+        try {
+            final int prestartedNodes = nodeCount() + 1;
 
-        CountDownLatch latch = new CountDownLatch(nodeCount());
+            String name = "serviceOnEachNodeButClientUpdateTopology";
 
-        DummyService.exeLatch(name, latch);
+            Ignite g = randomGrid();
 
-        IgniteServices svcs = g.services().withAsync();
+            CountDownLatch latch = new CountDownLatch(nodeCount());
 
-        svcs.deployNodeSingleton(name, new DummyService());
+            DummyService.exeLatch(name, latch);
 
-        IgniteFuture<?> fut = svcs.future();
+            IgniteServices svcs = g.services().withAsync();
 
-        info("Deployed service: " + name);
+            svcs.deployNodeSingleton(name, new DummyService());
 
-        fut.get();
+            IgniteFuture<?> fut = svcs.future();
 
-        info("Finished waiting for service future: " + name);
+            info("Deployed service: " + name);
 
-        latch.await();
+            fut.get();
 
-        TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
-        TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            info("Finished waiting for service future: " + name);
 
-        int newNodes = 2;
+            latch.await();
 
-        latch = new CountDownLatch(newNodes);
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
 
-        DummyService.exeLatch(name, latch);
+            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
+            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+            int servers = 2;
+            int clients = 2;
+
+            latch = new CountDownLatch(servers);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                // Ensure service is deployed
+                assertNotNull(grid(prestartedNodes + servers - 1)
+                    .services().serviceProxy(name, Service.class, false, 2000));
+
+                TestCase.assertEquals(name, nodeCount() + servers, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
+            }
+            finally {
+                stopExtraNodes(servers + clients);
+            }
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
 
-        startExtraNodes(newNodes);
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachNodeUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
         try {
+            String name = "serviceOnEachNodeUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            final int prestartedNodes = nodeCount() + 1;
+
+            CountDownLatch latch = new CountDownLatch(prestartedNodes);
+
+            DummyService.exeLatch(name, latch);
+
+            ServiceConfiguration srvcCfg = new ServiceConfiguration();
+
+            srvcCfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
+            srvcCfg.setName(name);
+            srvcCfg.setMaxPerNodeCount(1);
+            srvcCfg.setService(new DummyService());
+
+            IgniteServices svcs = g.services().withAsync();
+
+            svcs.deploy(srvcCfg);
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
             latch.await();
 
-            TestCase.assertEquals(name, nodeCount() + newNodes, DummyService.started(name));
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+            TestCase.assertEquals(name, prestartedNodes, DummyService.started(name));
             TestCase.assertEquals(name, 0, DummyService.cancelled(name));
 
-            checkCount(name, g.services().serviceDescriptors(), nodeCount() + newNodes);
+            int servers = 2;
+            int clients = 2;
+
+            int extraNodes = servers + clients;
+
+            latch = new CountDownLatch(extraNodes);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                // Ensure service is deployed
+                assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+                TestCase.assertEquals(name, prestartedNodes + extraNodes, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), prestartedNodes + extraNodes);
+            }
+            finally {
+                stopExtraNodes(extraNodes);
+            }
         }
         finally {
-            stopExtraNodes(newNodes);
+            stopGrid("client");
         }
     }
 }
\ No newline at end of file

From b214211eb3461746b6931c0623e086bb208e5dda Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Thu, 6 Apr 2017 15:00:12 +0300
Subject: [PATCH 092/516] IGNITE-4644 Value from IgniteQueue in atomic mode
 could be lost

---
 .../java/org/apache/ignite/IgniteSystemProperties.java     | 5 +++++
 .../datastructures/GridAtomicCacheQueueImpl.java           | 7 +++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index f0270986fc4b5..2f88d47a4c285 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -304,6 +304,11 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_ATOMIC_DEFERRED_ACK_TIMEOUT = "IGNITE_ATOMIC_DEFERRED_ACK_TIMEOUT";
 
+    /**
+     * Atomic cache deferred update timeout.
+     */
+    public static final String IGNITE_ATOMIC_CACHE_QUEUE_RETRY_TIMEOUT = "IGNITE_ATOMIC_CACHE_QUEUE_RETRY_TIMEOUT";
+
     /**
      * One phase commit deferred ack request timeout.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridAtomicCacheQueueImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridAtomicCacheQueueImpl.java
index 58d3efefd7605..3cee09e0be28a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridAtomicCacheQueueImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridAtomicCacheQueueImpl.java
@@ -28,12 +28,14 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_ATOMIC_CACHE_QUEUE_RETRY_TIMEOUT;
+
 /**
  * {@link org.apache.ignite.IgniteQueue} implementation using atomic cache.
  */
 public class GridAtomicCacheQueueImpl<T> extends GridCacheQueueAdapter<T> {
     /** */
-    private static final long RETRY_TIMEOUT = 3000;
+    private static final long RETRY_TIMEOUT = Integer.getInteger(IGNITE_ATOMIC_CACHE_QUEUE_RETRY_TIMEOUT, 10000);
 
     /**
      * @param queueName Queue name.
@@ -94,7 +96,8 @@ public GridAtomicCacheQueueImpl(String queueName, GridCacheQueueHeader hdr, Grid
                         return data;
                 }
 
-                U.warn(log, "Failed to get item, will retry poll [queue=" + queueName + ", idx=" + idx + ']');
+                U.warn(log, "Failed to get item due to poll timeout [queue=" + queueName + ", idx=" + idx + "]. " +
+                    "Poll timeout can be redefined by 'IGNITE_ATOMIC_CACHE_QUEUE_RETRY_TIMEOUT' system property.");
             }
         }
         catch (IgniteCheckedException e) {

From 014161427fb603b6df7c8ecc3c0904f5df47a21d Mon Sep 17 00:00:00 2001
From: Denis Magda <dmagda@gridgain.com>
Date: Mon, 13 Feb 2017 20:33:32 -0500
Subject: [PATCH 093/516] IGNITE-4159: Kubernetes IP finder. (cherry picked
 from commit 37c0a22)

---
 modules/kubernetes/DEVNOTES.txt               |  63 ++++
 modules/kubernetes/README.txt                 |  33 ++
 modules/kubernetes/config/Dockerfile          |  45 +++
 modules/kubernetes/config/example-kube.xml    |  44 +++
 .../kubernetes/config/ignite-deployment.yaml  |  26 ++
 modules/kubernetes/config/ignite-service.yaml |  14 +
 modules/kubernetes/config/run.sh              |  50 +++
 modules/kubernetes/licenses/apache-2.0.txt    | 202 +++++++++++
 modules/kubernetes/pom.xml                    |  93 ++++++
 .../TcpDiscoveryKubernetesIpFinder.java       | 315 ++++++++++++++++++
 .../tcp/ipfinder/kubernetes/package-info.java |  22 ++
 ...cpDiscoveryKubernetesIpFinderSelfTest.java |  93 ++++++
 .../tcp/ipfinder/kubernetes/package-info.java |  22 ++
 .../testsuites/IgniteKubernetesTestSuite.java |  41 +++
 pom.xml                                       |   1 +
 15 files changed, 1064 insertions(+)
 create mode 100644 modules/kubernetes/DEVNOTES.txt
 create mode 100644 modules/kubernetes/README.txt
 create mode 100644 modules/kubernetes/config/Dockerfile
 create mode 100644 modules/kubernetes/config/example-kube.xml
 create mode 100644 modules/kubernetes/config/ignite-deployment.yaml
 create mode 100644 modules/kubernetes/config/ignite-service.yaml
 create mode 100644 modules/kubernetes/config/run.sh
 create mode 100644 modules/kubernetes/licenses/apache-2.0.txt
 create mode 100644 modules/kubernetes/pom.xml
 create mode 100644 modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
 create mode 100644 modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
 create mode 100644 modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinderSelfTest.java
 create mode 100644 modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
 create mode 100644 modules/kubernetes/src/test/java/org/apache/ignite/testsuites/IgniteKubernetesTestSuite.java

diff --git a/modules/kubernetes/DEVNOTES.txt b/modules/kubernetes/DEVNOTES.txt
new file mode 100644
index 0000000000000..b2a8173587bf7
--- /dev/null
+++ b/modules/kubernetes/DEVNOTES.txt
@@ -0,0 +1,63 @@
+Building and testing Kubernetes module
+=========================================
+
+The instructions provide a guidance on how to build and test Ignite Kubernetes IP finder in Kubernetes environment.
+
+To test the IP finder you have to build the whole Apache Ignite project, package the binary as a Docker image and
+feed the image to your kubernetes environment.
+
+Building Apache Ignite
+=========================
+
+Use the command below to assemble an Apache Ignite binary:
+    mvn clean package -Prelease -Dignite.edition=fabric-lgpl -DskipTests
+
+Note, if you alter the build instruction somehow make sure to update the files under 'config' folder if needed.
+
+Installing Docker and Minikube
+==============================
+
+Install Docker and Minikube for testing purpose in your development environment.
+
+Once this is done, make sure that Minikube sees Docker images registered locally:
+    eval $(minikube docker-env)
+
+Start Minikube:
+    minikube start --vm-driver=xhyve
+
+Assembling Apache Ignite Docker Image
+=====================================
+
+Create a folder for all the files to be placed in the Docker image and copy the following there:
+    - Apache Ignite binary in a zip form built at the step above.
+    - Dockerfile from `ignite-kubernetes/config/Dockerfile`.
+    - Ignite configuration with enabled Kubernetes IP finder from `ignite-kubernetes/config/example-kube.xml`.
+    - The executable file that will start an Ignite node process from `ignite-kubernetes/config/run.sh`
+
+Go to the folder and execute a command below to prepare the image:
+    docker build -t ignite-kube:v1 .
+
+Creating containerized Ignite pods and Ignite lookup service
+============================================================
+
+Start the Kubernetes service that is used for IP addresses lookup. Use `ignite-kubernetes/config/ignite-service.yaml`:
+	kubectl create -f {path_to}/ignite-service.yaml
+
+Create and deploy Ignite pods using `ignite-kubernetes/config/ignite-deployment.yaml` configuration:
+    kubectl create -f {path_to}/ignite-deployment.yaml
+
+Make sure that the pods were deployed and running properly:
+    kubectl get pod
+    kubectl logs {pod name}
+
+Increase or decrease number of Ignite pods checking that Kubernetes IP finder works as expected:
+    kubectl scale --replicas=4 -f {path_to}/ignite-deployment.yaml
+
+Docker Image Redeployment
+=========================
+
+If you need to redeploy the docker image after it gets updated and you prefer not to change the image version then
+delete a current Kubernetes Ignite deployment (don't delete the service):
+    kubectl delete deployment ignite-cluster
+
+After that you are free to build and deploy an updated docker image using the same commands as listed above.
diff --git a/modules/kubernetes/README.txt b/modules/kubernetes/README.txt
new file mode 100644
index 0000000000000..a9a5a092ce2bb
--- /dev/null
+++ b/modules/kubernetes/README.txt
@@ -0,0 +1,33 @@
+Apache Ignite Kubernetes Module
+------------------------
+
+Apache Ignite Kubernetes module provides a TCP Discovery IP Finder that uses a dedicated Kubernetes service
+for IP addresses lookup of Apache Ignite pods containerized by Kubernetes.
+
+To enable Kubernetes module when starting a standalone node, move 'optional/ignite-kubernetes' folder to
+'libs' folder before running 'ignite.{sh|bat}' script. The content of the module folder will
+be added to classpath in this case.
+
+Importing Kubernetes Module In Maven Project
+-------------------------------------
+
+If you are using Maven to manage dependencies of your project, you can add Kubernetes module
+dependency like this (replace '${ignite.version}' with actual Ignite version you are
+interested in):
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                        http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    ...
+    <dependencies>
+        ...
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-kubernetes</artifactId>
+            <version>${ignite.version}</version>
+        </dependency>
+        ...
+    </dependencies>
+    ...
+</project>
diff --git a/modules/kubernetes/config/Dockerfile b/modules/kubernetes/config/Dockerfile
new file mode 100644
index 0000000000000..4e08ce80e2147
--- /dev/null
+++ b/modules/kubernetes/config/Dockerfile
@@ -0,0 +1,45 @@
+# Use Java 8 image as default one.
+FROM java:8
+
+# Set Apache Ignite version.
+ENV IGNITE_VERSION 2.0.0-SNAPSHOT
+
+# Set IGNITE_HOME variable.
+ENV IGNITE_HOME /opt/ignite/apache-ignite-fabric-lgpl-${IGNITE_VERSION}-bin
+
+# Setting a path to an Apache Ignite configuration file. Used by run.sh script below.
+ENV CONFIG_URI ${IGNITE_HOME}/config/example-kube.xml
+
+# Make sure kubernetes lib is copied to 'libs' folder.
+ENV OPTION_LIBS ignite-kubernetes
+
+# Disabling quiet mode.
+ENV IGNITE_QUIET=false
+
+# Install or update needed tools.
+RUN apt-get update && apt-get install -y --no-install-recommends unzip
+
+# Creating and setting a working directory for following commands.
+WORKDIR /opt/ignite
+
+# Copying local Apache Ignite build to the docker image.
+COPY ./apache-ignite-fabric-lgpl-${IGNITE_VERSION}-bin.zip apache-ignite-fabric-lgpl-${IGNITE_VERSION}-bin.zip
+
+# Unpacking the build.
+RUN unzip apache-ignite-fabric-lgpl-${IGNITE_VERSION}-bin.zip
+RUN rm apache-ignite-fabric-lgpl-${IGNITE_VERSION}-bin.zip
+
+# Copying the executable file and setting permissions.
+COPY ./run.sh $IGNITE_HOME/
+RUN chmod +x $IGNITE_HOME/run.sh
+
+# Copying the configuration.
+COPY ./example-kube.xml $IGNITE_HOME/config
+
+# Starting an Apache Ignite node.
+CMD $IGNITE_HOME/run.sh
+
+# Exposing the ports.
+EXPOSE 11211 47100 47500 49112
+
+
diff --git a/modules/kubernetes/config/example-kube.xml b/modules/kubernetes/config/example-kube.xml
new file mode 100644
index 0000000000000..bc04463f71a88
--- /dev/null
+++ b/modules/kubernetes/config/example-kube.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    Configuration example with Kubernetes IP finder enabled.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+    <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+
+        <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. -->
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <!--
+                        Enables Kubernetes IP finder with default settings.
+                    -->
+                    <bean
+                        class="org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder"/>
+                </property>
+            </bean>
+        </property>
+    </bean>
+</beans>
diff --git a/modules/kubernetes/config/ignite-deployment.yaml b/modules/kubernetes/config/ignite-deployment.yaml
new file mode 100644
index 0000000000000..ed5c102a666a6
--- /dev/null
+++ b/modules/kubernetes/config/ignite-deployment.yaml
@@ -0,0 +1,26 @@
+# An example of a Kubernetes configuration for Ignite pods deployment.
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  # Custom Ignite cluster's name.
+  name: ignite-cluster
+spec:
+  # Number of nodes to be started by Kubernetes initially.
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        # Must be equal to Ignite's Kubernetes service name.
+        app: ignite
+    spec:
+      containers:
+        # Custom Ignite node's pod  name.
+      - name: ignite-node
+        # Custom Ignite Docker image name.
+        image: ignite-kube:v1
+        ports:
+        # Ports to open.
+        - containerPort: 11211 # REST port number.
+        - containerPort: 47100 # communication SPI port number.
+        - containerPort: 47500 # discovery SPI port number.
+        - containerPort: 49112 # JMX port number.
diff --git a/modules/kubernetes/config/ignite-service.yaml b/modules/kubernetes/config/ignite-service.yaml
new file mode 100644
index 0000000000000..07b751624cbbf
--- /dev/null
+++ b/modules/kubernetes/config/ignite-service.yaml
@@ -0,0 +1,14 @@
+# An example of a Kubernetes configuration for Ignite lookup service deployment.
+apiVersion: v1
+kind: Service
+metadata:
+  # Name of Ignite Service used by Kubernetes IP finder for IP addresses lookup.
+  # The name must be equal to TcpDiscoveryKubernetesIpFinder.setServiceName parameter.
+  name: ignite
+spec:
+  clusterIP: None
+  ports:
+    - port: 9042 # some custom port (optional).
+  selector:
+    # Must be equal to the label set for Ignite pods.
+    app: ignite
diff --git a/modules/kubernetes/config/run.sh b/modules/kubernetes/config/run.sh
new file mode 100644
index 0000000000000..dbf287118aa41
--- /dev/null
+++ b/modules/kubernetes/config/run.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ ! -z "$OPTION_LIBS" ]; then
+  IFS=, LIBS_LIST=("$OPTION_LIBS")
+
+  for lib in ${LIBS_LIST[@]}; do
+    cp -r $IGNITE_HOME/libs/optional/"$lib"/* \
+        $IGNITE_HOME/libs/
+  done
+fi
+
+if [ ! -z "$EXTERNAL_LIBS" ]; then
+  IFS=, LIBS_LIST=("$EXTERNAL_LIBS")
+
+  for lib in ${LIBS_LIST[@]}; do
+    echo $lib >> temp
+  done
+
+  wget -i temp -P $IGNITE_HOME/libs
+
+  rm temp
+fi
+
+QUIET=""
+
+if [ "$IGNITE_QUIET" = "false" ]; then
+  QUIET="-v"
+fi
+
+if [ -z $CONFIG_URI ]; then
+  $IGNITE_HOME/bin/ignite.sh $QUIET
+else
+  $IGNITE_HOME/bin/ignite.sh $QUIET $CONFIG_URI
+fi
diff --git a/modules/kubernetes/licenses/apache-2.0.txt b/modules/kubernetes/licenses/apache-2.0.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/modules/kubernetes/licenses/apache-2.0.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/modules/kubernetes/pom.xml b/modules/kubernetes/pom.xml
new file mode 100644
index 0000000000000..5d4e5f0c8b509
--- /dev/null
+++ b/modules/kubernetes/pom.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    POM file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.ignite</groupId>
+        <artifactId>ignite-parent</artifactId>
+        <version>1</version>
+        <relativePath>../../parent</relativePath>
+    </parent>
+
+    <artifactId>ignite-kubernetes</artifactId>
+    <version>2.0.0-SNAPSHOT</version>
+    <url>http://ignite.apache.org</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.codehaus.jackson</groupId>
+            <artifactId>jackson-core-asl</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.codehaus.jackson</groupId>
+            <artifactId>jackson-mapper-asl</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-core</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-beans</artifactId>
+            <version>${spring.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+            <version>${spring.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-core</artifactId>
+            <version>${spring.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>
diff --git a/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
new file mode 100644
index 0000000000000..a5bd24f0affcc
--- /dev/null
+++ b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes;
+
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+
+/**
+ * IP finder for automatic lookup of Ignite nodes running in Kubernetes environment. All Ignite nodes have to deployed
+ * as Kubernetes pods in order to be discovered. An application that uses Ignite client nodes as a gateway to the
+ * cluster is required to be containerized as well. Applications and Ignite nodes running outside of Kubernetes will
+ * not be able to reach the containerized counterparts.
+ * <p>
+ * The implementation is based on a dedicated Kubernetes service that has to be created and should be deployed prior
+ * Ignite nodes startup. The service will maintain a list of all endpoints (internal IP addresses) of all containerized
+ * Ignite pods running so far. The name of the service must be equal to {@link #setServiceName(String)} which is
+ * `ignite` by default.
+ * <p>
+ * As for Ignite pods, it's recommended to label them in such a way that the service will use the label in its selector
+ * configuration excluding endpoints of irrelevant Kubernetes pods running in parallel.
+ * <p>
+ * The IP finder, in its turn, will call this service to retrieve Ignite pods IP addresses. The port will be
+ * either the one that is set with {@link TcpDiscoverySpi#setLocalPort(int)} or {@link TcpDiscoverySpi#DFLT_PORT}.
+ * Make sure that all Ignite pods occupy a similar discovery port, otherwise they will not be able to discover each
+ * other using this IP finder.
+ * <h2 class="header">Optional configuration</h2>
+ * <ul>
+ *      <li>The Kubernetes service name for IP addresses lookup (see {@link #setServiceName(String)})</li>
+ *      <li>The Kubernetes service namespace for IP addresses lookup (see {@link #setNamespace(String)}</li>
+ *      <li>The host name of the Kubernetes API server (see {@link #setMasterUrl(String)})</li>
+ *      <li>Path to the service token (see {@link #setAccountToken(String)}</li>
+ * </ul>
+ * <p>
+ * Both {@link #registerAddresses(Collection)} and {@link #unregisterAddresses(Collection)} have no effect.
+ * <p>
+ * Note, this IP finder is only workable when it used in Kubernetes environment.
+ * Choose another implementation of {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} for local
+ * or home network tests.
+ */
+public class TcpDiscoveryKubernetesIpFinder extends TcpDiscoveryIpFinderAdapter {
+    /** Grid logger. */
+    @LoggerResource
+    private IgniteLogger log;
+
+    /** Init routine guard. */
+    private final AtomicBoolean initGuard = new AtomicBoolean();
+
+    /** Init routine latch. */
+    private final CountDownLatch initLatch = new CountDownLatch(1);
+
+    /** Trust manager. */
+    private TrustManager[] trustAll = new TrustManager[] {
+        new X509TrustManager() {
+            public void checkServerTrusted(X509Certificate[] certs, String authType) {}
+            public void checkClientTrusted(X509Certificate[] certs, String authType) {}
+            public X509Certificate[] getAcceptedIssuers() { return null; }
+        }
+    };
+
+    /** Host verifier. */
+    private HostnameVerifier trustAllHosts = new HostnameVerifier() {
+        public boolean verify(String hostname, SSLSession session) {
+            return true;
+        }
+    };
+
+    /** Ignite's Kubernetes Service name. */
+    private String serviceName = "ignite";
+
+    /** Ignite Pod setNamespace name. */
+    private String namespace = "default";
+
+    /** Kubernetes API server URL in a string form. */
+    private String master = "https://kubernetes.default.svc.cluster.local:443";
+
+    /** Account token location. */
+    private String accountToken = "/var/run/secrets/kubernetes.io/serviceaccount/token";
+
+    /** Kubernetes API server URL. */
+    private URL url;
+
+    /** SSL context */
+    private SSLContext ctx;
+
+    /**
+     * Creates an instance of Kubernetes IP finder.
+     */
+    public TcpDiscoveryKubernetesIpFinder() {
+        setShared(true);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException {
+        init();
+
+        Collection<InetSocketAddress> addrs = new ArrayList<>();
+
+        try {
+            System.out.println("Getting Apache Ignite endpoints from: " + url);
+
+            HttpsURLConnection conn = (HttpsURLConnection)url.openConnection();
+
+            conn.setHostnameVerifier(trustAllHosts);
+
+            conn.setSSLSocketFactory(ctx.getSocketFactory());
+            conn.addRequestProperty("Authorization", "Bearer " + serviceAccountToken(accountToken));
+
+            // Sending the request and processing a response.
+            ObjectMapper mapper = new ObjectMapper();
+
+            Endpoints endpoints = mapper.readValue(conn.getInputStream(), Endpoints.class);
+
+            if (endpoints != null) {
+                if (endpoints.subsets != null && !endpoints.subsets.isEmpty()) {
+                    for (Subset subset : endpoints.subsets) {
+
+                        if (subset.addresses != null && !subset.addresses.isEmpty()) {
+                            for (Address address : subset.addresses) {
+                                addrs.add(new InetSocketAddress(address.ip, 0));
+
+                                System.out.println("Added an address to the list: " + address.ip);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        catch (Exception e) {
+            throw new IgniteSpiException("Failed to retrieve Ignite pods IP addresses.", e);
+        }
+
+        return addrs;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unregisterAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
+        // No-op
+    }
+
+    /**
+     * Sets the name of Kubernetes service for Ignite pods' IP addresses lookup. The name of the service must be equal
+     * to the name set in service's Kubernetes configuration. If this parameter is not changed then the name of the
+     * service has to be set to 'ignite' in the corresponding Kubernetes configuration.
+     *
+     * @param service Kubernetes service name for IP addresses lookup. If it's not set then 'ignite' is used by default.
+     */
+    public void setServiceName(String service) {
+        this.serviceName = service;
+    }
+
+    /**
+     * Sets the namespace the Kubernetes service belongs to. By default, it's supposed that the service is running under
+     * Kubernetes `default` namespace.
+     *
+     * @param namespace The Kubernetes service namespace for IP addresses lookup.
+     */
+    public void setNamespace(String namespace) {
+        this.namespace = namespace;
+    }
+
+    /**
+     * Sets the host name of the Kubernetes API server. By default the following host name is used:
+     * 'https://kubernetes.default.svc.cluster.local:443'.
+     *
+     * @param master The host name of the Kubernetes API server.
+     */
+    public void setMasterUrl(String master) {
+        this.master = master;
+    }
+
+    /**
+     * Specifies the path to the service token file. By default the following account token is used:
+     * '/var/run/secrets/kubernetes.io/serviceaccount/token'.
+     *
+     * @param accountToken The path to the service token file.
+     */
+    public void setAccountToken(String accountToken) {
+        this.accountToken = accountToken;
+    }
+
+    /**
+     * Kubernetes IP finder initialization.
+     *
+     * @throws IgniteSpiException In case of error.
+     */
+    private void init() throws IgniteSpiException {
+        if (initGuard.compareAndSet(false, true)) {
+
+            if (serviceName == null || serviceName.isEmpty() ||
+                namespace == null || namespace.isEmpty() ||
+                master == null || master.isEmpty() ||
+                accountToken == null || accountToken.isEmpty()) {
+                throw new IgniteSpiException(
+                    "One or more configuration parameters are invalid [setServiceName=" +
+                        serviceName + ", setNamespace=" + namespace + ", setMasterUrl=" +
+                        master + ", setAccountToken=" + accountToken + "]");
+            }
+
+            try {
+                // Preparing the URL and SSL context to be used for connection purposes.
+                String path = String.format("/api/v1/namespaces/%s/endpoints/%s", namespace, serviceName);
+
+                url = new URL(master + path);
+
+                ctx = SSLContext.getInstance("SSL");
+
+                ctx.init(null, trustAll, new SecureRandom());
+            }
+            catch (Exception e) {
+                throw new IgniteSpiException("Failed to connect to Ignite's Kubernetes Service.", e);
+            }
+            finally {
+                initLatch.countDown();
+            }
+        }
+        else {
+            try {
+                U.await(initLatch);
+            }
+            catch (IgniteInterruptedCheckedException e) {
+                throw new IgniteSpiException("Thread has been interrupted.", e);
+            }
+
+            if (url == null || ctx == null)
+                throw new IgniteSpiException("IP finder has not been initialized properly.");
+        }
+    }
+
+    /**
+     * Reads content of the service account token file.
+     *
+     * @param file The path to the service account token.
+     * @return Service account token.
+     */
+    private String serviceAccountToken(String file)  {
+        try {
+            return new String(Files.readAllBytes(Paths.get(file)));
+        } catch (IOException e) {
+            throw new IgniteSpiException("Failed to load services account token [setAccountToken= " + file + "]", e);
+        }
+    }
+
+    /**
+     * Object used by Jackson for processing of Kubernetes lookup service's response.
+     */
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private static class Address {
+        /** */
+        public String ip;
+    }
+
+    /**
+     * Object used by Jackson for processing of Kubernetes lookup service's response.
+     */
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private static class Subset {
+        /** */
+        public List<Address> addresses;
+    }
+
+    /**
+     * Object used by Jackson for processing of Kubernetes lookup service's response.
+     */
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private static class Endpoints {
+        /** */
+        public List<Subset> subsets;
+    }
+}
diff --git a/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
new file mode 100644
index 0000000000000..a572cb2d8cbc1
--- /dev/null
+++ b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains Kubernetes IP finder implementation.
+ */
+package org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes;
\ No newline at end of file
diff --git a/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinderSelfTest.java b/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinderSelfTest.java
new file mode 100644
index 0000000000000..fd3e2a35e432c
--- /dev/null
+++ b/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinderSelfTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes;
+
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAbstractSelfTest;
+
+/**
+ * TcpDiscoveryKubernetesIpFinder test.
+ */
+public class TcpDiscoveryKubernetesIpFinderSelfTest extends
+    TcpDiscoveryIpFinderAbstractSelfTest<TcpDiscoveryKubernetesIpFinder> {
+    /**
+     * Constructor.
+     *
+     * @throws Exception If any error occurs.
+     */
+    public TcpDiscoveryKubernetesIpFinderSelfTest() throws Exception {
+        // No-op.
+    }
+
+    @Override protected void beforeTest() throws Exception {
+        // No-op.
+    }
+
+    /* {@inheritDoc} */
+    @Override protected TcpDiscoveryKubernetesIpFinder ipFinder() throws Exception {
+        // No-op.
+        return null;
+    }
+
+    /* {@inheritDoc} */
+    @Override public void testIpFinder() throws Exception {
+        TcpDiscoveryKubernetesIpFinder ipFinder = new TcpDiscoveryKubernetesIpFinder();
+
+        ipFinder.setAccountToken(null);
+
+        try {
+            ipFinder.getRegisteredAddresses();
+        }
+        catch (IgniteSpiException e) {
+            assertTrue(e.getMessage().startsWith("One or more configuration parameters are invalid"));
+        }
+
+        ipFinder = new TcpDiscoveryKubernetesIpFinder();
+
+        ipFinder.setMasterUrl(null);
+
+        try {
+            ipFinder.getRegisteredAddresses();
+        }
+        catch (IgniteSpiException e) {
+            assertTrue(e.getMessage().startsWith("One or more configuration parameters are invalid"));
+        }
+
+        ipFinder = new TcpDiscoveryKubernetesIpFinder();
+
+        ipFinder.setNamespace(null);
+
+        try {
+            ipFinder.getRegisteredAddresses();
+        }
+        catch (IgniteSpiException e) {
+            assertTrue(e.getMessage().startsWith("One or more configuration parameters are invalid"));
+        }
+
+        ipFinder = new TcpDiscoveryKubernetesIpFinder();
+
+        ipFinder.setServiceName("");
+
+        try {
+            ipFinder.getRegisteredAddresses();
+        }
+        catch (IgniteSpiException e) {
+            assertTrue(e.getMessage().startsWith("One or more configuration parameters are invalid"));
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java b/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
new file mode 100644
index 0000000000000..83ab56ff75163
--- /dev/null
+++ b/modules/kubernetes/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains Kubernetes IP finder internal tests.
+ */
+package org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes;
\ No newline at end of file
diff --git a/modules/kubernetes/src/test/java/org/apache/ignite/testsuites/IgniteKubernetesTestSuite.java b/modules/kubernetes/src/test/java/org/apache/ignite/testsuites/IgniteKubernetesTestSuite.java
new file mode 100644
index 0000000000000..540657e8b2601
--- /dev/null
+++ b/modules/kubernetes/src/test/java/org/apache/ignite/testsuites/IgniteKubernetesTestSuite.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+
+import junit.framework.TestSuite;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinderSelfTest;
+import org.apache.ignite.testframework.IgniteTestSuite;
+
+/**
+ * Ignite Kubernetes integration test.
+ */
+public class IgniteKubernetesTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        TestSuite suite = new IgniteTestSuite("Kubernetes Integration Test Suite");
+
+        // Cloud Nodes IP finder.
+        suite.addTestSuite(TcpDiscoveryKubernetesIpFinderSelfTest.class);
+
+        return suite;
+    }
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index d83bb1b969b9e..ef8573d3d5390 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,6 +88,7 @@
         <module>modules/web/ignite-websphere-test</module>
         <module>modules/cassandra</module>
         <module>modules/flink</module>
+        <module>modules/kubernetes</module>
     </modules>
 
     <profiles>

From 1db238402f11c67d2b28bfb7ff47955415f00c25 Mon Sep 17 00:00:00 2001
From: Denis Magda <dmagda@gridgain.com>
Date: Wed, 15 Feb 2017 23:37:26 -0500
Subject: [PATCH 094/516] IGNITE-4159: fixing logging (cherry picked from
 commit 06908d2) (cherry picked from commit fa27ee3)

---
 .../kubernetes/TcpDiscoveryKubernetesIpFinder.java        | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
index a5bd24f0affcc..53b6df6a0b711 100644
--- a/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
+++ b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
@@ -52,7 +52,7 @@
  * cluster is required to be containerized as well. Applications and Ignite nodes running outside of Kubernetes will
  * not be able to reach the containerized counterparts.
  * <p>
- * The implementation is based on a dedicated Kubernetes service that has to be created and should be deployed prior
+ * The implementation is based on a distinct Kubernetes service that has to be created and should be deployed prior
  * Ignite nodes startup. The service will maintain a list of all endpoints (internal IP addresses) of all containerized
  * Ignite pods running so far. The name of the service must be equal to {@link #setServiceName(String)} which is
  * `ignite` by default.
@@ -137,7 +137,8 @@ public TcpDiscoveryKubernetesIpFinder() {
         Collection<InetSocketAddress> addrs = new ArrayList<>();
 
         try {
-            System.out.println("Getting Apache Ignite endpoints from: " + url);
+            if (log.isDebugEnabled())
+                log.debug("Getting Apache Ignite endpoints from: " + url);
 
             HttpsURLConnection conn = (HttpsURLConnection)url.openConnection();
 
@@ -159,7 +160,8 @@ public TcpDiscoveryKubernetesIpFinder() {
                             for (Address address : subset.addresses) {
                                 addrs.add(new InetSocketAddress(address.ip, 0));
 
-                                System.out.println("Added an address to the list: " + address.ip);
+                                if (log.isDebugEnabled())
+                                    log.debug("Added an address to the list: " + address.ip);
                             }
                         }
                     }

From 5dfe16f7e91374008b9f6dfbb899364f5b2e1164 Mon Sep 17 00:00:00 2001
From: Denis Magda <dmagda@gridgain.com>
Date: Tue, 14 Feb 2017 01:03:30 -0500
Subject: [PATCH 095/516] IGNITE-4159: using logger instead of
 system.out.println (cherry picked from commit b9bf77c)

---
 modules/kubernetes/config/example-kube.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/kubernetes/config/example-kube.xml b/modules/kubernetes/config/example-kube.xml
index bc04463f71a88..11309d8933b14 100644
--- a/modules/kubernetes/config/example-kube.xml
+++ b/modules/kubernetes/config/example-kube.xml
@@ -41,4 +41,4 @@
             </bean>
         </property>
     </bean>
-</beans>
+</beans>
\ No newline at end of file

From 6e596d1ef426b66abd866d011a8f5cf5c5c25124 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 6 Apr 2017 14:43:50 +0300
Subject: [PATCH 096/516] IGNITE-4832: Prevent service deployment on client by
 default when configuration is provided on startup. This closes #1748.

(cherry picked from commit b7ab273)
---
 .../service/GridServiceProcessor.java         |  10 +-
 .../GridServiceProcessorAbstractSelfTest.java |  11 ++
 ...rviceProcessorMultiNodeConfigSelfTest.java |  74 +++++++++-
 ...GridServiceProcessorMultiNodeSelfTest.java | 139 +++++++++++++++---
 4 files changed, 202 insertions(+), 32 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index adfbc116aa293..d0b2733473333 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -70,11 +70,11 @@
 import org.apache.ignite.internal.processors.cache.query.CacheQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
-import org.apache.ignite.internal.processors.continuous.AbstractContinuousMessage;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridEmptyIterator;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.SerializableTransient;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -92,7 +92,6 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
-import org.apache.ignite.internal.util.SerializableTransient;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
@@ -296,8 +295,13 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         if (cfgs != null) {
             Collection<IgniteInternalFuture<?>> futs = new ArrayList<>();
 
-            for (ServiceConfiguration c : ctx.config().getServiceConfiguration())
+            for (ServiceConfiguration c : cfgs) {
+                // Deploy only on server nodes by default.
+                if (c.getNodeFilter() == null)
+                    c.setNodeFilter(ctx.cluster().get().forServers().predicate());
+
                 futs.add(deploy(c));
+            }
 
             // Await for services to deploy.
             for (IgniteInternalFuture<?> f : futs)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
index 111cb714e8f32..0f79855518e3d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
@@ -129,6 +129,17 @@ protected void startExtraNodes(int cnt) throws Exception {
             startGrid(nodeCount() + i);
     }
 
+    /** */
+    protected void startExtraNodes(int servers, int clients) throws Exception {
+        startExtraNodes(servers);
+
+        for (int i = 0; i < clients; i++) {
+            final String nodeName = getTestGridName(nodeCount() + servers + i);
+
+            startGrid(nodeName, getConfiguration(nodeName).setClientMode(true));
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index b819cc93e6fb8..1bd3b035a446c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -19,7 +19,9 @@
 
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
+import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -33,6 +35,9 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
     /** Node singleton name. */
     private static final String NODE_SINGLE = "serviceConfigEachNode";
 
+    /** Node singleton name. */
+    private static final String NODE_SINGLE_BUT_CLIENT = "serviceConfigEachNodeButClient";
+
     /** Affinity service name. */
     private static final String AFFINITY = "serviceConfigAffinity";
 
@@ -46,7 +51,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        ServiceConfiguration[] arr = new ServiceConfiguration[3];
+        ServiceConfiguration[] arr = new ServiceConfiguration[4];
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -59,7 +64,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
         cfg = new ServiceConfiguration();
 
-        cfg.setName(NODE_SINGLE);
+        cfg.setName(NODE_SINGLE_BUT_CLIENT);
         cfg.setMaxPerNodeCount(1);
         cfg.setService(new DummyService());
 
@@ -76,6 +81,15 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
         arr[2] = cfg;
 
+        cfg = new ServiceConfiguration();
+
+        cfg.setName(NODE_SINGLE);
+        cfg.setMaxPerNodeCount(1);
+        cfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
+        cfg.setService(new DummyService());
+
+        arr[3] = cfg;
+
         return arr;
     }
 
@@ -91,6 +105,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
                         DummyService.cancelled(CLUSTER_SINGLE) == 0 &&
                         DummyService.started(NODE_SINGLE) == nodeCount() &&
                         DummyService.cancelled(NODE_SINGLE) == 0 &&
+                        DummyService.started(NODE_SINGLE_BUT_CLIENT) == nodeCount() &&
+                        DummyService.cancelled(NODE_SINGLE_BUT_CLIENT) == 0 &&
                         actualCount(AFFINITY, randomGrid().services().serviceDescriptors()) == 1;
                 }
             },
@@ -112,6 +128,13 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
         checkDeployOnEachNodeUpdateTopology(NODE_SINGLE);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
+        checkDeployOnEachNodeButClientUpdateTopology(NODE_SINGLE_BUT_CLIENT);
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -120,6 +143,10 @@ public void testAll() throws Exception {
 
         DummyService.reset();
 
+        checkDeployOnEachNodeButClientUpdateTopology(NODE_SINGLE_BUT_CLIENT);
+
+        DummyService.reset();
+
         checkDeployOnEachNodeUpdateTopology(NODE_SINGLE);
 
         DummyService.reset();
@@ -152,9 +179,7 @@ public void testAffinityUpdateTopology() throws Exception {
     private void checkSingletonUpdateTopology(String name) throws Exception {
         Ignite g = randomGrid();
 
-        int nodeCnt = 2;
-
-        startExtraNodes(nodeCnt);
+        startExtraNodes(2, 2);
 
         try {
             assertEquals(name, 0, DummyService.started(name));
@@ -165,7 +190,7 @@ private void checkSingletonUpdateTopology(String name) throws Exception {
             checkCount(name, g.services().serviceDescriptors(), 1);
         }
         finally {
-            stopExtraNodes(nodeCnt);
+            stopExtraNodes(4);
         }
     }
 
@@ -176,17 +201,21 @@ private void checkSingletonUpdateTopology(String name) throws Exception {
     private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         Ignite g = randomGrid();
 
-        int newNodes = 2;
+        int newNodes = 4;
 
         CountDownLatch latch = new CountDownLatch(newNodes);
 
         DummyService.exeLatch(name, latch);
 
-        startExtraNodes(newNodes);
+        startExtraNodes(2, 2);
 
         try {
             latch.await();
 
+            // Ensure service is deployed.
+            assertNotNull(grid(nodeCount() + newNodes - 1).services()
+                .serviceProxy(NODE_SINGLE_BUT_CLIENT, Service.class, false, 2000));
+
             assertEquals(name, newNodes, DummyService.started(name));
             assertEquals(name, 0, DummyService.cancelled(name));
 
@@ -196,4 +225,33 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
             stopExtraNodes(newNodes);
         }
     }
+
+    /**
+     * @param name Name.
+     * @throws Exception If failed.
+     */
+    private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Exception {
+        Ignite g = randomGrid();
+
+        int servers = 2;
+        int clients = 2;
+
+        CountDownLatch latch = new CountDownLatch(servers);
+
+        DummyService.exeLatch(name, latch);
+
+        startExtraNodes(servers, clients);
+
+        try {
+            latch.await();
+
+            assertEquals(name, servers, DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
+        }
+        finally {
+            stopExtraNodes(servers + clients);
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index 39336ef001a02..f7403dcc820ca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -21,7 +21,10 @@
 import junit.framework.TestCase;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteServices;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
 
 /**
  * Single node services test.
@@ -121,50 +124,144 @@ public void testAffinityDeployUpdateTopology() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testDeployOnEachNodeUpdateTopology() throws Exception {
-        String name = "serviceOnEachNodeUpdateTopology";
+    public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
-        Ignite g = randomGrid();
+        try {
+            final int prestartedNodes = nodeCount() + 1;
 
-        CountDownLatch latch = new CountDownLatch(nodeCount());
+            String name = "serviceOnEachNodeButClientUpdateTopology";
 
-        DummyService.exeLatch(name, latch);
+            Ignite g = randomGrid();
 
-        IgniteServices svcs = g.services().withAsync();
+            CountDownLatch latch = new CountDownLatch(nodeCount());
 
-        svcs.deployNodeSingleton(name, new DummyService());
+            DummyService.exeLatch(name, latch);
 
-        IgniteFuture<?> fut = svcs.future();
+            IgniteServices svcs = g.services().withAsync();
 
-        info("Deployed service: " + name);
+            svcs.deployNodeSingleton(name, new DummyService());
 
-        fut.get();
+            IgniteFuture<?> fut = svcs.future();
 
-        info("Finished waiting for service future: " + name);
+            info("Deployed service: " + name);
 
-        latch.await();
+            fut.get();
 
-        TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
-        TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            info("Finished waiting for service future: " + name);
 
-        int newNodes = 2;
+            latch.await();
 
-        latch = new CountDownLatch(newNodes);
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
 
-        DummyService.exeLatch(name, latch);
+            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
+            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+            int servers = 2;
+            int clients = 2;
+
+            latch = new CountDownLatch(servers);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                // Ensure service is deployed
+                assertNotNull(grid(prestartedNodes + servers - 1)
+                    .services().serviceProxy(name, Service.class, false, 2000));
+
+                TestCase.assertEquals(name, nodeCount() + servers, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
+            }
+            finally {
+                stopExtraNodes(servers + clients);
+            }
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
 
-        startExtraNodes(newNodes);
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachNodeUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
         try {
+            String name = "serviceOnEachNodeUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            final int prestartedNodes = nodeCount() + 1;
+
+            CountDownLatch latch = new CountDownLatch(prestartedNodes);
+
+            DummyService.exeLatch(name, latch);
+
+            ServiceConfiguration srvcCfg = new ServiceConfiguration();
+
+            srvcCfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
+            srvcCfg.setName(name);
+            srvcCfg.setMaxPerNodeCount(1);
+            srvcCfg.setService(new DummyService());
+
+            IgniteServices svcs = g.services().withAsync();
+
+            svcs.deploy(srvcCfg);
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
             latch.await();
 
-            TestCase.assertEquals(name, nodeCount() + newNodes, DummyService.started(name));
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+            TestCase.assertEquals(name, prestartedNodes, DummyService.started(name));
             TestCase.assertEquals(name, 0, DummyService.cancelled(name));
 
-            checkCount(name, g.services().serviceDescriptors(), nodeCount() + newNodes);
+            int servers = 2;
+            int clients = 2;
+
+            int extraNodes = servers + clients;
+
+            latch = new CountDownLatch(extraNodes);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                // Ensure service is deployed
+                assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+                TestCase.assertEquals(name, prestartedNodes + extraNodes, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), prestartedNodes + extraNodes);
+            }
+            finally {
+                stopExtraNodes(extraNodes);
+            }
         }
         finally {
-            stopExtraNodes(newNodes);
+            stopGrid("client");
         }
     }
 }
\ No newline at end of file

From 443ac9a7aa82af1359a03bcfc8f9212b108300e4 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 5 Apr 2017 15:01:02 +0300
Subject: [PATCH 097/516] IGNITE-4917: Fixed failure when accessing
 BinaryObjectBuilder field value serialized with OptimizedMarshaller . This
 closes #1736.

---
 .../internal/binary/builder/BinaryBuilderReader.java     | 8 ++++++++
 .../binary/mutabletest/GridBinaryTestClasses.java        | 9 +++++++++
 2 files changed, 17 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
index baaabd6a1a0bd..02264e3311529 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
@@ -476,6 +476,14 @@ public Object getValueQuickly(int pos, int len) {
                 return new BinaryPlainBinaryObject(binaryObj);
             }
 
+            case GridBinaryMarshaller.OPTM_MARSH: {
+                final BinaryHeapInputStream bin = BinaryHeapInputStream.create(arr, pos + 1);
+
+                final Object obj = BinaryUtils.doReadOptimized(bin, ctx, U.resolveClassLoader(ctx.configuration()));
+
+                return obj;
+            }
+
             default:
                 throw new BinaryObjectException("Invalid flag value: " + type);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
index 5ddb87dcefb31..0d4de7fa0e215 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
@@ -24,6 +24,7 @@
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Date;
@@ -141,6 +142,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Integer i_;
 
+        /** */
+        public BigInteger bi_;
+
         /** */
         public Long l_;
 
@@ -150,6 +154,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Double d_;
 
+        /** */
+        public BigDecimal bd_;
+
         /** */
         public Character c_;
 
@@ -267,9 +274,11 @@ public void setDefaultData() {
             b_ = 11;
             s_ = 22;
             i_ = 33;
+            bi_ = new BigInteger("33000000000000");
             l_ = 44L;
             f_ = 55f;
             d_ = 66d;
+            bd_ = new BigDecimal("33000000000000.123456789");
             c_ = 'e';
             z_ = true;
 

From 05f3c747921aed6838804d2f5f2c8d2bd7985337 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 5 Apr 2017 15:01:02 +0300
Subject: [PATCH 098/516] IGNITE-4917: Fixed failure when accessing
 BinaryObjectBuilder field value serialized with OptimizedMarshaller . This
 closes #1736.

(cherry picked from commit 443ac9a)
---
 .../internal/binary/builder/BinaryBuilderReader.java     | 8 ++++++++
 .../binary/mutabletest/GridBinaryTestClasses.java        | 9 +++++++++
 2 files changed, 17 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
index baaabd6a1a0bd..02264e3311529 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
@@ -476,6 +476,14 @@ public Object getValueQuickly(int pos, int len) {
                 return new BinaryPlainBinaryObject(binaryObj);
             }
 
+            case GridBinaryMarshaller.OPTM_MARSH: {
+                final BinaryHeapInputStream bin = BinaryHeapInputStream.create(arr, pos + 1);
+
+                final Object obj = BinaryUtils.doReadOptimized(bin, ctx, U.resolveClassLoader(ctx.configuration()));
+
+                return obj;
+            }
+
             default:
                 throw new BinaryObjectException("Invalid flag value: " + type);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
index 5ddb87dcefb31..0d4de7fa0e215 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
@@ -24,6 +24,7 @@
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Date;
@@ -141,6 +142,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Integer i_;
 
+        /** */
+        public BigInteger bi_;
+
         /** */
         public Long l_;
 
@@ -150,6 +154,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Double d_;
 
+        /** */
+        public BigDecimal bd_;
+
         /** */
         public Character c_;
 
@@ -267,9 +274,11 @@ public void setDefaultData() {
             b_ = 11;
             s_ = 22;
             i_ = 33;
+            bi_ = new BigInteger("33000000000000");
             l_ = 44L;
             f_ = 55f;
             d_ = 66d;
+            bd_ = new BigDecimal("33000000000000.123456789");
             c_ = 'e';
             z_ = true;
 

From b8e3d1b6f972b6d30657b7c85c8b34506dc29b88 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 5 Apr 2017 15:01:02 +0300
Subject: [PATCH 099/516] IGNITE-4917: Fixed failure when accessing
 BinaryObjectBuilder field value serialized with OptimizedMarshaller . This
 closes #1736.

(cherry picked from commit 443ac9a)
---
 .../internal/binary/builder/BinaryBuilderReader.java     | 8 ++++++++
 .../binary/mutabletest/GridBinaryTestClasses.java        | 9 +++++++++
 2 files changed, 17 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
index baaabd6a1a0bd..02264e3311529 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
@@ -476,6 +476,14 @@ public Object getValueQuickly(int pos, int len) {
                 return new BinaryPlainBinaryObject(binaryObj);
             }
 
+            case GridBinaryMarshaller.OPTM_MARSH: {
+                final BinaryHeapInputStream bin = BinaryHeapInputStream.create(arr, pos + 1);
+
+                final Object obj = BinaryUtils.doReadOptimized(bin, ctx, U.resolveClassLoader(ctx.configuration()));
+
+                return obj;
+            }
+
             default:
                 throw new BinaryObjectException("Invalid flag value: " + type);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
index 5ddb87dcefb31..0d4de7fa0e215 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
@@ -24,6 +24,7 @@
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Date;
@@ -141,6 +142,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Integer i_;
 
+        /** */
+        public BigInteger bi_;
+
         /** */
         public Long l_;
 
@@ -150,6 +154,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Double d_;
 
+        /** */
+        public BigDecimal bd_;
+
         /** */
         public Character c_;
 
@@ -267,9 +274,11 @@ public void setDefaultData() {
             b_ = 11;
             s_ = 22;
             i_ = 33;
+            bi_ = new BigInteger("33000000000000");
             l_ = 44L;
             f_ = 55f;
             d_ = 66d;
+            bd_ = new BigDecimal("33000000000000.123456789");
             c_ = 'e';
             z_ = true;
 

From a0392605f39c23fcd20c98d852c4cab749cd059b Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 6 Apr 2017 14:43:50 +0300
Subject: [PATCH 100/516] IGNITE-4832: Prevent service deployment on client by
 default when configuration is provided on startup. This closes #1748.

(cherry picked from commit b7ab273)
---
 .../service/GridServiceProcessor.java         |  10 +-
 .../GridServiceProcessorAbstractSelfTest.java |  11 ++
 ...rviceProcessorMultiNodeConfigSelfTest.java |  74 +++++++++-
 ...GridServiceProcessorMultiNodeSelfTest.java | 139 +++++++++++++++---
 4 files changed, 202 insertions(+), 32 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index adfbc116aa293..d0b2733473333 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -70,11 +70,11 @@
 import org.apache.ignite.internal.processors.cache.query.CacheQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
-import org.apache.ignite.internal.processors.continuous.AbstractContinuousMessage;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridEmptyIterator;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.SerializableTransient;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -92,7 +92,6 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
-import org.apache.ignite.internal.util.SerializableTransient;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
@@ -296,8 +295,13 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         if (cfgs != null) {
             Collection<IgniteInternalFuture<?>> futs = new ArrayList<>();
 
-            for (ServiceConfiguration c : ctx.config().getServiceConfiguration())
+            for (ServiceConfiguration c : cfgs) {
+                // Deploy only on server nodes by default.
+                if (c.getNodeFilter() == null)
+                    c.setNodeFilter(ctx.cluster().get().forServers().predicate());
+
                 futs.add(deploy(c));
+            }
 
             // Await for services to deploy.
             for (IgniteInternalFuture<?> f : futs)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
index 111cb714e8f32..0f79855518e3d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
@@ -129,6 +129,17 @@ protected void startExtraNodes(int cnt) throws Exception {
             startGrid(nodeCount() + i);
     }
 
+    /** */
+    protected void startExtraNodes(int servers, int clients) throws Exception {
+        startExtraNodes(servers);
+
+        for (int i = 0; i < clients; i++) {
+            final String nodeName = getTestGridName(nodeCount() + servers + i);
+
+            startGrid(nodeName, getConfiguration(nodeName).setClientMode(true));
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index b819cc93e6fb8..1bd3b035a446c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -19,7 +19,9 @@
 
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
+import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -33,6 +35,9 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
     /** Node singleton name. */
     private static final String NODE_SINGLE = "serviceConfigEachNode";
 
+    /** Node singleton name. */
+    private static final String NODE_SINGLE_BUT_CLIENT = "serviceConfigEachNodeButClient";
+
     /** Affinity service name. */
     private static final String AFFINITY = "serviceConfigAffinity";
 
@@ -46,7 +51,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        ServiceConfiguration[] arr = new ServiceConfiguration[3];
+        ServiceConfiguration[] arr = new ServiceConfiguration[4];
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -59,7 +64,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
         cfg = new ServiceConfiguration();
 
-        cfg.setName(NODE_SINGLE);
+        cfg.setName(NODE_SINGLE_BUT_CLIENT);
         cfg.setMaxPerNodeCount(1);
         cfg.setService(new DummyService());
 
@@ -76,6 +81,15 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
         arr[2] = cfg;
 
+        cfg = new ServiceConfiguration();
+
+        cfg.setName(NODE_SINGLE);
+        cfg.setMaxPerNodeCount(1);
+        cfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
+        cfg.setService(new DummyService());
+
+        arr[3] = cfg;
+
         return arr;
     }
 
@@ -91,6 +105,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
                         DummyService.cancelled(CLUSTER_SINGLE) == 0 &&
                         DummyService.started(NODE_SINGLE) == nodeCount() &&
                         DummyService.cancelled(NODE_SINGLE) == 0 &&
+                        DummyService.started(NODE_SINGLE_BUT_CLIENT) == nodeCount() &&
+                        DummyService.cancelled(NODE_SINGLE_BUT_CLIENT) == 0 &&
                         actualCount(AFFINITY, randomGrid().services().serviceDescriptors()) == 1;
                 }
             },
@@ -112,6 +128,13 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
         checkDeployOnEachNodeUpdateTopology(NODE_SINGLE);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
+        checkDeployOnEachNodeButClientUpdateTopology(NODE_SINGLE_BUT_CLIENT);
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -120,6 +143,10 @@ public void testAll() throws Exception {
 
         DummyService.reset();
 
+        checkDeployOnEachNodeButClientUpdateTopology(NODE_SINGLE_BUT_CLIENT);
+
+        DummyService.reset();
+
         checkDeployOnEachNodeUpdateTopology(NODE_SINGLE);
 
         DummyService.reset();
@@ -152,9 +179,7 @@ public void testAffinityUpdateTopology() throws Exception {
     private void checkSingletonUpdateTopology(String name) throws Exception {
         Ignite g = randomGrid();
 
-        int nodeCnt = 2;
-
-        startExtraNodes(nodeCnt);
+        startExtraNodes(2, 2);
 
         try {
             assertEquals(name, 0, DummyService.started(name));
@@ -165,7 +190,7 @@ private void checkSingletonUpdateTopology(String name) throws Exception {
             checkCount(name, g.services().serviceDescriptors(), 1);
         }
         finally {
-            stopExtraNodes(nodeCnt);
+            stopExtraNodes(4);
         }
     }
 
@@ -176,17 +201,21 @@ private void checkSingletonUpdateTopology(String name) throws Exception {
     private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         Ignite g = randomGrid();
 
-        int newNodes = 2;
+        int newNodes = 4;
 
         CountDownLatch latch = new CountDownLatch(newNodes);
 
         DummyService.exeLatch(name, latch);
 
-        startExtraNodes(newNodes);
+        startExtraNodes(2, 2);
 
         try {
             latch.await();
 
+            // Ensure service is deployed.
+            assertNotNull(grid(nodeCount() + newNodes - 1).services()
+                .serviceProxy(NODE_SINGLE_BUT_CLIENT, Service.class, false, 2000));
+
             assertEquals(name, newNodes, DummyService.started(name));
             assertEquals(name, 0, DummyService.cancelled(name));
 
@@ -196,4 +225,33 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
             stopExtraNodes(newNodes);
         }
     }
+
+    /**
+     * @param name Name.
+     * @throws Exception If failed.
+     */
+    private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Exception {
+        Ignite g = randomGrid();
+
+        int servers = 2;
+        int clients = 2;
+
+        CountDownLatch latch = new CountDownLatch(servers);
+
+        DummyService.exeLatch(name, latch);
+
+        startExtraNodes(servers, clients);
+
+        try {
+            latch.await();
+
+            assertEquals(name, servers, DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
+        }
+        finally {
+            stopExtraNodes(servers + clients);
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index 39336ef001a02..f7403dcc820ca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -21,7 +21,10 @@
 import junit.framework.TestCase;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteServices;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
 
 /**
  * Single node services test.
@@ -121,50 +124,144 @@ public void testAffinityDeployUpdateTopology() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testDeployOnEachNodeUpdateTopology() throws Exception {
-        String name = "serviceOnEachNodeUpdateTopology";
+    public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
-        Ignite g = randomGrid();
+        try {
+            final int prestartedNodes = nodeCount() + 1;
 
-        CountDownLatch latch = new CountDownLatch(nodeCount());
+            String name = "serviceOnEachNodeButClientUpdateTopology";
 
-        DummyService.exeLatch(name, latch);
+            Ignite g = randomGrid();
 
-        IgniteServices svcs = g.services().withAsync();
+            CountDownLatch latch = new CountDownLatch(nodeCount());
 
-        svcs.deployNodeSingleton(name, new DummyService());
+            DummyService.exeLatch(name, latch);
 
-        IgniteFuture<?> fut = svcs.future();
+            IgniteServices svcs = g.services().withAsync();
 
-        info("Deployed service: " + name);
+            svcs.deployNodeSingleton(name, new DummyService());
 
-        fut.get();
+            IgniteFuture<?> fut = svcs.future();
 
-        info("Finished waiting for service future: " + name);
+            info("Deployed service: " + name);
 
-        latch.await();
+            fut.get();
 
-        TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
-        TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            info("Finished waiting for service future: " + name);
 
-        int newNodes = 2;
+            latch.await();
 
-        latch = new CountDownLatch(newNodes);
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
 
-        DummyService.exeLatch(name, latch);
+            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
+            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+            int servers = 2;
+            int clients = 2;
+
+            latch = new CountDownLatch(servers);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                // Ensure service is deployed
+                assertNotNull(grid(prestartedNodes + servers - 1)
+                    .services().serviceProxy(name, Service.class, false, 2000));
+
+                TestCase.assertEquals(name, nodeCount() + servers, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
+            }
+            finally {
+                stopExtraNodes(servers + clients);
+            }
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
 
-        startExtraNodes(newNodes);
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachNodeUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
         try {
+            String name = "serviceOnEachNodeUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            final int prestartedNodes = nodeCount() + 1;
+
+            CountDownLatch latch = new CountDownLatch(prestartedNodes);
+
+            DummyService.exeLatch(name, latch);
+
+            ServiceConfiguration srvcCfg = new ServiceConfiguration();
+
+            srvcCfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
+            srvcCfg.setName(name);
+            srvcCfg.setMaxPerNodeCount(1);
+            srvcCfg.setService(new DummyService());
+
+            IgniteServices svcs = g.services().withAsync();
+
+            svcs.deploy(srvcCfg);
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
             latch.await();
 
-            TestCase.assertEquals(name, nodeCount() + newNodes, DummyService.started(name));
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+            TestCase.assertEquals(name, prestartedNodes, DummyService.started(name));
             TestCase.assertEquals(name, 0, DummyService.cancelled(name));
 
-            checkCount(name, g.services().serviceDescriptors(), nodeCount() + newNodes);
+            int servers = 2;
+            int clients = 2;
+
+            int extraNodes = servers + clients;
+
+            latch = new CountDownLatch(extraNodes);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                // Ensure service is deployed
+                assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+                TestCase.assertEquals(name, prestartedNodes + extraNodes, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), prestartedNodes + extraNodes);
+            }
+            finally {
+                stopExtraNodes(extraNodes);
+            }
         }
         finally {
-            stopExtraNodes(newNodes);
+            stopGrid("client");
         }
     }
 }
\ No newline at end of file

From 220db882b466c03eadd148b3a19a0bf70d82d4a6 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 10 Apr 2017 10:28:15 +0300
Subject: [PATCH 101/516] IGNITE-2466 - Use current NIO back pressure mechanism
 to limit received messages. Mark them process only when backups acknowledged.

---
 .../managers/communication/GridIoManager.java |  12 +-
 .../dht/atomic/GridDhtAtomicCache.java        |  23 ++-
 .../util/nio/GridNioBackPressureControl.java  |  39 ++++-
 .../util/nio/GridNioMessageTracker.java       |   7 +
 ...acheAtomicPrimarySyncBackPressureTest.java | 151 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   3 +
 6 files changed, 220 insertions(+), 15 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index bb3add4833ada..dbd5db6730175 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -738,7 +738,7 @@ private void processP2PMessage(
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     GridMessageListener lsnr = listenerGet0(msg.topic());
 
@@ -752,7 +752,7 @@ private void processP2PMessage(
                     invokeListener(msg.policy(), lsnr, nodeId, obj);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
 
                     msgC.run();
                 }
@@ -787,12 +787,12 @@ private void processRegularMessage(
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     processRegularMessage0(msg, nodeId);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
 
                     msgC.run();
                 }
@@ -1148,12 +1148,12 @@ else if (log.isDebugEnabled()) {
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     unwindMessageSet(msgSet0, lsnr);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
                 }
             }
         };
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 463fc57d2115f..047be8751d7b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -84,6 +84,8 @@
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.nio.GridNioBackPressureControl;
+import org.apache.ignite.internal.util.nio.GridNioMessageTracker;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -92,14 +94,15 @@
 import org.apache.ignite.internal.util.typedef.CX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteOutClosure;
+import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
@@ -110,6 +113,7 @@
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
@@ -1904,8 +1908,23 @@ private void updateAllAsyncInternal0(
                         if (req.writeSynchronizationMode() != FULL_ASYNC)
                             req.cleanup(!node.isLocal());
 
-                        if (dhtFut != null)
+                        if (dhtFut != null) {
+                            if (req.writeSynchronizationMode() == PRIMARY_SYNC && !dhtFut.isDone()) {
+                                final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
+
+                                if (tracker != null && tracker instanceof GridNioMessageTracker) {
+                                    ((GridNioMessageTracker)tracker).onMessageReceived();
+
+                                    dhtFut.listen(new IgniteInClosure<IgniteInternalFuture<Void>>() {
+                                        @Override public void apply(IgniteInternalFuture<Void> fut) {
+                                            ((GridNioMessageTracker)tracker).onMessageProcessed();
+                                        }
+                                    });
+                                }
+                            }
+
                             ctx.mvcc().addAtomicFuture(dhtFut.version(), dhtFut);
+                        }
                     }
                     else
                         // Should remap all keys.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
index 96a1ab347d402..37d985f7ff100 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
@@ -17,14 +17,17 @@
 
 package org.apache.ignite.internal.util.nio;
 
+import org.apache.ignite.lang.IgniteRunnable;
+import org.jetbrains.annotations.Nullable;
+
 /**
  * Utility class that allows to ignore back-pressure control for threads that are processing messages.
  */
 public class GridNioBackPressureControl {
     /** Thread local flag indicating that thread is processing message. */
-    private static ThreadLocal<Boolean> threadProcMsg = new ThreadLocal<Boolean>() {
-        @Override protected Boolean initialValue() {
-            return Boolean.FALSE;
+    private static ThreadLocal<Holder> threadProcMsg = new ThreadLocal<Holder>() {
+        @Override protected Holder initialValue() {
+            return new Holder();
         }
     };
 
@@ -32,13 +35,35 @@ public class GridNioBackPressureControl {
      * @return Flag indicating whether current thread is processing message.
      */
     public static boolean threadProcessingMessage() {
-        return threadProcMsg.get();
+        return threadProcMsg.get().procMsg;
     }
 
     /**
      * @param processing Flag indicating whether current thread is processing message.
+     * @param tracker Thread local back pressure tracker of messages, associated with one connection.
+     */
+    public static void threadProcessingMessage(boolean processing, @Nullable IgniteRunnable tracker) {
+        Holder holder = threadProcMsg.get();
+
+        holder.procMsg = processing;
+        holder.tracker = tracker;
+    }
+
+    /**
+     * @return Thread local back pressure tracker of messages, associated with one connection.
      */
-    public static void threadProcessingMessage(boolean processing) {
-        threadProcMsg.set(processing);
+    @Nullable public static IgniteRunnable threadTracker() {
+        return threadProcMsg.get().tracker;
+    }
+
+    /**
+     *
+     */
+    private static class Holder {
+        /** Process message. */
+        private boolean procMsg;
+
+        /** Tracker. */
+        private IgniteRunnable tracker;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
index e02c7ca7fd6ee..f05ee0c60d045 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
@@ -92,6 +92,13 @@ public GridNioMessageTracker(GridNioSession ses, int msgQueueLimit) {
         }
     }
 
+    /**
+     *
+     */
+    public void onMessageProcessed() {
+        run();
+    }
+
     /**
      */
     public void onMessageReceived() {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
new file mode 100644
index 0000000000000..49e3e5ca07a15
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Checks that back-pressure control restricts uncontrolled growing
+ * of backup message queue. This means, if queue too big - any reads
+ * will be stopped until received acks from backup nodes.
+ */
+public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = new CacheConfiguration("cache");
+
+        ccfg.setBackups(1);
+
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
+        ccfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED);
+        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+        TestCommunicationSpi spi = new TestCommunicationSpi();
+
+        spi.setMessageQueueLimit(100);
+
+        cfg.setCommunicationSpi(spi);
+        cfg.setClientMode(gridName.contains("client"));
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientPut() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+
+        final Ignite client = startGrid("client");
+
+        checkBackPressure(client, srv1, srv2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testServerPut() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+
+        final Ignite client = startGrid("server3");
+
+        checkBackPressure(client, srv1, srv2);
+    }
+
+    /**
+     * @param client Producer node.
+     * @throws InterruptedException If failed.
+     */
+    private void checkBackPressure(Ignite client, final Ignite srv1, final Ignite srv2) throws Exception {
+        final IgniteCache<Integer, String> cache = client.cache("cache");
+
+        awaitPartitionMapExchange();
+
+        for (int i = 0; i < 10000; i++) {
+            cache.put(i, String.valueOf(i));
+
+            if (i % 100 == 0) {
+                int size1 = futuresNum(srv1);
+                int size2 = futuresNum(srv2);
+
+                assert size1 < 150 : size1;
+                assert size2 < 150 : size2;
+            }
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @return Size of the backup queue.
+     */
+    private int futuresNum(Ignite ignite) {
+        return ((IgniteKernal)ignite).context().cache().context().mvcc().atomicFutures().size();
+    }
+
+    /**
+     * Delays backup update acks.
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (((GridIoMessage)msg).message() instanceof GridDhtAtomicDeferredUpdateResponse)
+                sleep(100);
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     * @param millis Millis.
+     */
+    private static void sleep(long millis) {
+        try {
+            Thread.sleep(millis);
+        }
+        catch (InterruptedException e) {
+            throw new IgniteSpiException(e);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 5a09a1c5ae797..9fcf31a322f2a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -94,6 +94,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteSystemCacheOnClientTest;
 import org.apache.ignite.internal.processors.cache.MarshallerCacheJobRunNodeRestartTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheAffinityEarlyTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheGetFutureHangsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheNoValueClassOnServerNodeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheCreatePutMultiNodeSelfTest;
@@ -334,6 +335,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteCacheNearOnlyTxTest.class);
 
+        suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class);
+
         return suite;
     }
 }
\ No newline at end of file

From 3d616799efb472227b3b313516e6b40729654631 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 10 Apr 2017 10:36:11 +0300
Subject: [PATCH 102/516] IGNITE-2466 - Use current NIO back pressure mechanism
 to limit received messages. Mark them process only when backups acknowledged.

(backport from 1.9.2)

(cherry picked from commit 220db882b466c03eadd148b3a19a0bf70d82d4a6)
---
 .../managers/communication/GridIoManager.java |  12 +-
 .../dht/atomic/GridDhtAtomicCache.java        |  23 ++-
 .../util/nio/GridNioBackPressureControl.java  |  39 ++++-
 .../util/nio/GridNioMessageTracker.java       |   7 +
 ...acheAtomicPrimarySyncBackPressureTest.java | 151 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   6 +
 6 files changed, 223 insertions(+), 15 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 2eda4b715ee01..a1315dbd96fb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -731,7 +731,7 @@ private void processP2PMessage(
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     GridMessageListener lsnr = listenerGet0(msg.topic());
 
@@ -745,7 +745,7 @@ private void processP2PMessage(
                     invokeListener(msg.policy(), lsnr, nodeId, obj);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
 
                     msgC.run();
                 }
@@ -780,12 +780,12 @@ private void processRegularMessage(
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     processRegularMessage0(msg, nodeId);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
 
                     msgC.run();
                 }
@@ -1141,12 +1141,12 @@ else if (log.isDebugEnabled()) {
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     unwindMessageSet(msgSet0, lsnr);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
                 }
             }
         };
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index c91f881b60378..7d360766dd0fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -84,6 +84,8 @@
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.nio.GridNioBackPressureControl;
+import org.apache.ignite.internal.util.nio.GridNioMessageTracker;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -92,14 +94,15 @@
 import org.apache.ignite.internal.util.typedef.CX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteOutClosure;
+import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
@@ -110,6 +113,7 @@
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
@@ -1908,8 +1912,23 @@ private void updateAllAsyncInternal0(
                         if (req.writeSynchronizationMode() != FULL_ASYNC)
                             req.cleanup(!node.isLocal());
 
-                        if (dhtFut != null)
+                        if (dhtFut != null) {
+                            if (req.writeSynchronizationMode() == PRIMARY_SYNC && !dhtFut.isDone()) {
+                                final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
+
+                                if (tracker != null && tracker instanceof GridNioMessageTracker) {
+                                    ((GridNioMessageTracker)tracker).onMessageReceived();
+
+                                    dhtFut.listen(new IgniteInClosure<IgniteInternalFuture<Void>>() {
+                                        @Override public void apply(IgniteInternalFuture<Void> fut) {
+                                            ((GridNioMessageTracker)tracker).onMessageProcessed();
+                                        }
+                                    });
+                                }
+                            }
+
                             ctx.mvcc().addAtomicFuture(dhtFut.version(), dhtFut);
+                        }
                     }
                     else
                         // Should remap all keys.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
index 96a1ab347d402..37d985f7ff100 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
@@ -17,14 +17,17 @@
 
 package org.apache.ignite.internal.util.nio;
 
+import org.apache.ignite.lang.IgniteRunnable;
+import org.jetbrains.annotations.Nullable;
+
 /**
  * Utility class that allows to ignore back-pressure control for threads that are processing messages.
  */
 public class GridNioBackPressureControl {
     /** Thread local flag indicating that thread is processing message. */
-    private static ThreadLocal<Boolean> threadProcMsg = new ThreadLocal<Boolean>() {
-        @Override protected Boolean initialValue() {
-            return Boolean.FALSE;
+    private static ThreadLocal<Holder> threadProcMsg = new ThreadLocal<Holder>() {
+        @Override protected Holder initialValue() {
+            return new Holder();
         }
     };
 
@@ -32,13 +35,35 @@ public class GridNioBackPressureControl {
      * @return Flag indicating whether current thread is processing message.
      */
     public static boolean threadProcessingMessage() {
-        return threadProcMsg.get();
+        return threadProcMsg.get().procMsg;
     }
 
     /**
      * @param processing Flag indicating whether current thread is processing message.
+     * @param tracker Thread local back pressure tracker of messages, associated with one connection.
+     */
+    public static void threadProcessingMessage(boolean processing, @Nullable IgniteRunnable tracker) {
+        Holder holder = threadProcMsg.get();
+
+        holder.procMsg = processing;
+        holder.tracker = tracker;
+    }
+
+    /**
+     * @return Thread local back pressure tracker of messages, associated with one connection.
      */
-    public static void threadProcessingMessage(boolean processing) {
-        threadProcMsg.set(processing);
+    @Nullable public static IgniteRunnable threadTracker() {
+        return threadProcMsg.get().tracker;
+    }
+
+    /**
+     *
+     */
+    private static class Holder {
+        /** Process message. */
+        private boolean procMsg;
+
+        /** Tracker. */
+        private IgniteRunnable tracker;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
index e02c7ca7fd6ee..f05ee0c60d045 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
@@ -92,6 +92,13 @@ public GridNioMessageTracker(GridNioSession ses, int msgQueueLimit) {
         }
     }
 
+    /**
+     *
+     */
+    public void onMessageProcessed() {
+        run();
+    }
+
     /**
      */
     public void onMessageReceived() {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
new file mode 100644
index 0000000000000..49e3e5ca07a15
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Checks that back-pressure control restricts uncontrolled growing
+ * of backup message queue. This means, if queue too big - any reads
+ * will be stopped until received acks from backup nodes.
+ */
+public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = new CacheConfiguration("cache");
+
+        ccfg.setBackups(1);
+
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
+        ccfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED);
+        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+        TestCommunicationSpi spi = new TestCommunicationSpi();
+
+        spi.setMessageQueueLimit(100);
+
+        cfg.setCommunicationSpi(spi);
+        cfg.setClientMode(gridName.contains("client"));
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientPut() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+
+        final Ignite client = startGrid("client");
+
+        checkBackPressure(client, srv1, srv2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testServerPut() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+
+        final Ignite client = startGrid("server3");
+
+        checkBackPressure(client, srv1, srv2);
+    }
+
+    /**
+     * @param client Producer node.
+     * @throws InterruptedException If failed.
+     */
+    private void checkBackPressure(Ignite client, final Ignite srv1, final Ignite srv2) throws Exception {
+        final IgniteCache<Integer, String> cache = client.cache("cache");
+
+        awaitPartitionMapExchange();
+
+        for (int i = 0; i < 10000; i++) {
+            cache.put(i, String.valueOf(i));
+
+            if (i % 100 == 0) {
+                int size1 = futuresNum(srv1);
+                int size2 = futuresNum(srv2);
+
+                assert size1 < 150 : size1;
+                assert size2 < 150 : size2;
+            }
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @return Size of the backup queue.
+     */
+    private int futuresNum(Ignite ignite) {
+        return ((IgniteKernal)ignite).context().cache().context().mvcc().atomicFutures().size();
+    }
+
+    /**
+     * Delays backup update acks.
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (((GridIoMessage)msg).message() instanceof GridDhtAtomicDeferredUpdateResponse)
+                sleep(100);
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     * @param millis Millis.
+     */
+    private static void sleep(long millis) {
+        try {
+            Thread.sleep(millis);
+        }
+        catch (InterruptedException e) {
+            throw new IgniteSpiException(e);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 2b446bbea8e6d..9fcf31a322f2a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -94,6 +94,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteSystemCacheOnClientTest;
 import org.apache.ignite.internal.processors.cache.MarshallerCacheJobRunNodeRestartTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheAffinityEarlyTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheGetFutureHangsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheNoValueClassOnServerNodeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheCreatePutMultiNodeSelfTest;
@@ -108,6 +109,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheMultiTxLockSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCrossCacheTxSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearTxPreloadSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheNearOnlyTxTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheNearReadCommittedTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridReplicatedTxPreloadTest;
 import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicLoadAllTest;
@@ -331,6 +333,10 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(MarshallerCacheJobRunNodeRestartTest.class);
 
+        suite.addTestSuite(IgniteCacheNearOnlyTxTest.class);
+
+        suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class);
+
         return suite;
     }
 }
\ No newline at end of file

From 2a88a7a7581465ff0a6f8733550e6d42d7f71a6c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 10 Apr 2017 10:54:37 +0300
Subject: [PATCH 103/516] IGNITE-4667 - Throw exception on starting client
 cache when indexed types cannot be loaded

---
 .../cache/CacheAffinitySharedManager.java     |  52 +--
 .../GridCachePartitionExchangeManager.java    |   4 +-
 .../processors/cache/GridCacheProcessor.java  |  42 ++-
 .../dht/atomic/GridDhtAtomicCache.java        |   3 +-
 .../GridDhtPartitionsExchangeFuture.java      |   2 +-
 ...niteClientCacheInitializationFailTest.java | 346 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   2 +
 7 files changed, 415 insertions(+), 36 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 2642d16c56fc5..84372febb2c91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -379,38 +379,48 @@ else if (req.start() && !req.clientStartOnly()) {
             Integer cacheId = CU.cacheId(req.cacheName());
 
             if (req.start()) {
-                cctx.cache().prepareCacheStart(req, fut.topologyVersion());
+                try {
+                    cctx.cache().prepareCacheStart(req, fut.topologyVersion());
 
-                if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
-                    if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
-                        U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
-                }
+                    if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
+                        if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
+                            U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+                    }
 
-                if (!crd || !lateAffAssign) {
-                    GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+                    if (!crd || !lateAffAssign) {
+                        GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
-                    if (cacheCtx != null && !cacheCtx.isLocal()) {
-                        boolean clientCacheStarted =
-                            req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
+                        if (cacheCtx != null && !cacheCtx.isLocal()) {
+                            boolean clientCacheStarted =
+                                req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
 
-                        if (clientCacheStarted)
-                            initAffinity(cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
-                        else if (!req.clientStartOnly()) {
-                            assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
+                            if (clientCacheStarted)
+                                initAffinity(cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
+                            else if (!req.clientStartOnly()) {
+                                assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
 
-                            GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
+                                GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
 
-                            assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
+                                assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
 
-                            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
-                                fut.discoveryEvent(), fut.discoCache());
+                                List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
+                                    fut.discoveryEvent(), fut.discoCache());
 
-                            aff.initialize(fut.topologyVersion(), assignment);
+                                aff.initialize(fut.topologyVersion(), assignment);
+                            }
                         }
                     }
+                    else
+                        initStartedCacheOnCoordinator(fut, cacheId);
+                }
+                catch (IgniteCheckedException | RuntimeException e) {
+                    U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
+                        "[cacheName=" + req.cacheName() + ']', e);
+
+                    cctx.cache().forceCloseCache(fut.topologyVersion(), req, e);
+
+                    throw e;
                 }
-                else
-                    initStartedCacheOnCoordinator(fut, cacheId);
             }
             else if (req.stop() || req.close()) {
                 cctx.cache().blockGateway(req);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 16b07694ed5c0..679395727123a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -241,11 +241,11 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                                     fut = affinityReadyFuture(req.cacheFutureTopologyVersion());
 
                                 if (fut == null || fut.isDone())
-                                    cctx.cache().completeStartFuture(req);
+                                    cctx.cache().completeStartFuture(req, null);
                                 else {
                                     fut.listen(new CI1<IgniteInternalFuture<?>>() {
                                         @Override public void apply(IgniteInternalFuture<?> fut) {
-                                            cctx.cache().completeStartFuture(req);
+                                            cctx.cache().completeStartFuture(req, null);
                                         }
                                     });
                                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 9487589aa4c05..5740ad17f0617 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1612,6 +1612,8 @@ public void prepareCacheStart(DynamicCacheChangeRequest req, AffinityTopologyVer
         assert req.start() : req;
         assert req.cacheType() != null : req;
 
+        String cacheName = maskNull(req.cacheName());
+
         prepareCacheStart(
             req.startCacheConfiguration(),
             req.nearCacheConfiguration(),
@@ -1622,7 +1624,7 @@ public void prepareCacheStart(DynamicCacheChangeRequest req, AffinityTopologyVer
             topVer
         );
 
-        DynamicCacheDescriptor desc = registeredCaches.get(maskNull(req.cacheName()));
+        DynamicCacheDescriptor desc = registeredCaches.get(cacheName);
 
         if (desc != null)
             desc.onStart();
@@ -1761,8 +1763,8 @@ private void stopGateway(DynamicCacheChangeRequest req) {
     /**
      * @param req Stop request.
      */
-    private void prepareCacheStop(DynamicCacheChangeRequest req) {
-        assert req.stop() || req.close() : req;
+    private void prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose) {
+        assert req.stop() || req.close() || forceClose : req;
 
         GridCacheAdapter<?, ?> cache = caches.remove(maskNull(req.cacheName()));
 
@@ -1779,18 +1781,35 @@ private void prepareCacheStop(DynamicCacheChangeRequest req) {
         }
     }
 
+    /**
+     * Closes cache even if it's not fully initialized (e.g. fail on cache init stage).
+     *
+     * @param topVer Completed topology version.
+     * @param req Change request.
+     * @param err Error.
+     */
+    void forceCloseCache(
+        AffinityTopologyVersion topVer,
+        DynamicCacheChangeRequest req,
+        Throwable err
+    ) {
+        onExchangeDone(topVer, Collections.singleton(req), err, true);
+    }
+
     /**
      * Callback invoked when first exchange future for dynamic cache is completed.
      *
      * @param topVer Completed topology version.
      * @param reqs Change requests.
      * @param err Error.
+     * @param forceClose Close cache despite flags in requests.
      */
     @SuppressWarnings("unchecked")
     public void onExchangeDone(
         AffinityTopologyVersion topVer,
         Collection<DynamicCacheChangeRequest> reqs,
-        Throwable err
+        Throwable err,
+        boolean forceClose
     ) {
         for (GridCacheAdapter<?, ?> cache : caches.values()) {
             GridCacheContext<?, ?> cacheCtx = cache.context();
@@ -1805,16 +1824,16 @@ public void onExchangeDone(
             }
         }
 
-        if (!F.isEmpty(reqs) && err == null) {
+        if (!F.isEmpty(reqs) && (err == null || forceClose)) {
             for (DynamicCacheChangeRequest req : reqs) {
                 String masked = maskNull(req.cacheName());
 
                 if (req.stop()) {
                     stopGateway(req);
 
-                    prepareCacheStop(req);
+                    prepareCacheStop(req, forceClose);
                 }
-                else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId())) {
+                else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId()) || forceClose) {
                     IgniteCacheProxy<?, ?> proxy = jCacheProxies.remove(masked);
 
                     if (proxy != null) {
@@ -1827,20 +1846,21 @@ else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId())) {
                         else {
                             proxy.context().gate().onStopped();
 
-                            prepareCacheStop(req);
+                            prepareCacheStop(req, forceClose);
                         }
                     }
                 }
 
-                completeStartFuture(req);
+                completeStartFuture(req, err);
             }
         }
     }
 
     /**
      * @param req Request to complete future for.
+     * @param err Error to be passed to futures.
      */
-    public void completeStartFuture(DynamicCacheChangeRequest req) {
+    public void completeStartFuture(DynamicCacheChangeRequest req, @Nullable Throwable err) {
         DynamicCacheStartFuture fut = (DynamicCacheStartFuture)pendingFuts.get(maskNull(req.cacheName()));
 
         assert req.deploymentId() != null;
@@ -1848,7 +1868,7 @@ public void completeStartFuture(DynamicCacheChangeRequest req) {
 
         if (fut != null && fut.deploymentId().equals(req.deploymentId()) &&
             F.eq(req.initiatingNodeId(), ctx.localNodeId()))
-            fut.onDone();
+            fut.onDone(err);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 7d360766dd0fc..5b9b1416d8f84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -457,7 +457,8 @@ else if (res.error() != null) {
 
     /** {@inheritDoc} */
     @Override public void stop() {
-        deferredUpdateMsgSnd.stop();
+        if (deferredUpdateMsgSnd != null)
+            deferredUpdateMsgSnd.stop();
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index f11f868cf0e33..9aa0755269b9f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1069,7 +1069,7 @@ private void sendPartitions(ClusterNode oldestNode) {
 
         cctx.exchange().onExchangeDone(this, err);
 
-        cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err);
+        cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err, false);
 
         if (super.onDone(res, err) && realExchange) {
             if (log.isDebugEnabled())
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
new file mode 100644
index 0000000000000..63dd57527183b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import javax.cache.Cache;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.query.GridQueryCancel;
+import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
+import org.apache.ignite.internal.processors.query.GridQueryIndexing;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.lang.GridCloseableIterator;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.indexing.IndexingQueryFilter;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Test checks whether cache initialization error on client side
+ * doesn't causes hangs and doesn't impact other caches.
+ */
+public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractTest {
+    /** Failed cache name. */
+    private static final String CACHE_NAME = "cache";
+
+    /** Atomic cache name. */
+    private static final String ATOMIC_CACHE_NAME = "atomic-cache";
+
+    /** Tx cache name. */
+    private static final String TX_CACHE_NAME = "tx-cache";
+
+    /** Near atomic cache name. */
+    private static final String NEAR_ATOMIC_CACHE_NAME = "near-atomic-cache";
+
+    /** Near tx cache name. */
+    private static final String NEAR_TX_CACHE_NAME = "near-tx-cache";
+
+    /** Failed caches. */
+    private static final Set<String> FAILED_CACHES;
+
+    static {
+        Set<String> set = new HashSet<>();
+
+        set.add(ATOMIC_CACHE_NAME);
+        set.add(TX_CACHE_NAME);
+        set.add(NEAR_ATOMIC_CACHE_NAME);
+        set.add(NEAR_TX_CACHE_NAME);
+
+        FAILED_CACHES = Collections.unmodifiableSet(set);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrid("server");
+        startGrid("client");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.contains("server")) {
+            CacheConfiguration<Integer, String> ccfg1 = new CacheConfiguration<>();
+
+            ccfg1.setIndexedTypes(Integer.class, String.class);
+            ccfg1.setName(ATOMIC_CACHE_NAME);
+            ccfg1.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+            CacheConfiguration<Integer, String> ccfg2 = new CacheConfiguration<>();
+
+            ccfg2.setIndexedTypes(Integer.class, String.class);
+            ccfg2.setName(TX_CACHE_NAME);
+            ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+            cfg.setCacheConfiguration(ccfg1, ccfg2);
+        }
+        else {
+            GridQueryProcessor.idxCls = FailedIndexing.class;
+
+            cfg.setClientMode(true);
+        }
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicCacheInitialization() throws Exception {
+        checkCacheInitialization(ATOMIC_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTransactionalCacheInitialization() throws Exception {
+        checkCacheInitialization(TX_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicNearCacheInitialization() throws Exception {
+        checkCacheInitialization(NEAR_ATOMIC_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTransactionalNearCacheInitialization() throws Exception {
+        checkCacheInitialization(NEAR_TX_CACHE_NAME);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkCacheInitialization(final String cacheName) throws Exception {
+        Ignite client = grid("client");
+
+        checkFailedCache(client, cacheName);
+
+        checkFineCache(client, CACHE_NAME + 1);
+
+        assertNull(client.cache(cacheName));
+        assertNull(client.getOrCreateCache(cacheName));
+
+        checkFineCache(client, CACHE_NAME + 2);
+    }
+
+    /**
+     * @param client Client.
+     * @param cacheName Cache name.
+     */
+    private void checkFineCache(Ignite client, String cacheName) {
+        IgniteCache<Integer, String> cache = client.getOrCreateCache(cacheName);
+
+        cache.put(1, "1");
+
+        assertEquals("1", cache.get(1));
+    }
+
+    /**
+     * @param client Client.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    private void checkFailedCache(final Ignite client, final String cacheName) {
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                IgniteCache<Integer, String> cache;
+
+                // Start cache with near enabled.
+                if (NEAR_ATOMIC_CACHE_NAME.equals(cacheName) || NEAR_TX_CACHE_NAME.equals(cacheName)) {
+                    CacheConfiguration<Integer, String> ccfg = new CacheConfiguration<Integer, String>(cacheName)
+                        .setNearConfiguration(new NearCacheConfiguration<Integer, String>());
+
+                    if (NEAR_TX_CACHE_NAME.equals(cacheName))
+                        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+                    cache = client.getOrCreateCache(ccfg);
+                }
+                else
+                    cache = client.cache(cacheName);
+
+
+                cache.put(1, "1");
+
+                assertEquals("1", cache.get(1));
+
+                return null;
+            }
+        }, CacheException.class, null);
+    }
+
+    /**
+     * To fail on cache start.
+     */
+    private static class FailedIndexing implements GridQueryIndexing {
+        /** {@inheritDoc} */
+        @Override public void start(GridKernalContext ctx, GridSpinBusyLock busyLock) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void stop() throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public QueryCursor<List<?>> queryTwoStep(GridCacheContext<?, ?> cctx, SqlFieldsQuery qry,
+            GridQueryCancel cancel) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public <K, V> QueryCursor<Cache.Entry<K, V>> queryTwoStep(GridCacheContext<?, ?> cctx,
+            SqlQuery qry) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public GridQueryFieldsResult queryLocalSqlFields(@Nullable String spaceName, String qry,
+            Collection<Object> params, IndexingQueryFilter filter, boolean enforceJoinOrder, int timeout,
+            GridQueryCancel cancel) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable String spaceName,
+            String qry, String alias, Collection<Object> params, GridQueryTypeDescriptor type,
+            IndexingQueryFilter filter) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalText(@Nullable String spaceName,
+            String qry, GridQueryTypeDescriptor type, IndexingQueryFilter filter) throws IgniteCheckedException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void registerCache(GridCacheContext<?, ?> cctx,
+            CacheConfiguration<?, ?> ccfg) throws IgniteCheckedException {
+            if (FAILED_CACHES.contains(cctx.name()) && cctx.kernalContext().clientNode())
+                throw new IgniteCheckedException("Test query exception " + cctx.name());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void unregisterCache(CacheConfiguration<?, ?> ccfg) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean registerType(@Nullable String spaceName,
+            GridQueryTypeDescriptor desc) throws IgniteCheckedException {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void unregisterType(@Nullable String spaceName,
+            GridQueryTypeDescriptor type) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void store(@Nullable String spaceName, GridQueryTypeDescriptor type, CacheObject key,
+            CacheObject val, byte[] ver, long expirationTime) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void remove(@Nullable String spaceName, CacheObject key,
+            CacheObject val) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onSwap(@Nullable String spaceName, CacheObject key) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onUnswap(@Nullable String spaceName, CacheObject key,
+            CacheObject val) throws IgniteCheckedException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void rebuildIndexes(@Nullable String spaceName, GridQueryTypeDescriptor type) {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public IndexingQueryFilter backupFilter(AffinityTopologyVersion topVer, int[] parts) {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onDisconnected(IgniteFuture<?> reconnectFut) {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public PreparedStatement prepareNativeStatement(String schema, String sql) throws SQLException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Collection<GridRunningQueryInfo> runningQueries(long duration) {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancelQueries(Collection<Long> queries) {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancelAllQueries() {
+
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 9fcf31a322f2a..914aedb40764a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -81,6 +81,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxPreloadNoWriteTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxReplicatedPeekModesTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxStoreValueTest;
+import org.apache.ignite.internal.processors.cache.IgniteClientCacheInitializationFailTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheFilterTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartNoExchangeTimeoutTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartSelfTest;
@@ -235,6 +236,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheAffinityEarlyTest.class);
         suite.addTestSuite(IgniteCacheCreatePutMultiNodeSelfTest.class);
         suite.addTestSuite(IgniteCacheCreatePutTest.class);
+        suite.addTestSuite(IgniteClientCacheInitializationFailTest.class);
 
         suite.addTestSuite(GridCacheTxLoadFromStoreOnLockSelfTest.class);
 

From ba6227be49c8a395a5632e9841a6acb65ae340b6 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 10 Apr 2017 11:40:17 +0300
Subject: [PATCH 104/516] IGNITE-2466 - Disable back-pressure for sender data
 nodes to avoid deadlock.

---
 .../dht/atomic/GridDhtAtomicCache.java           |  5 ++++-
 .../CacheAtomicPrimarySyncBackPressureTest.java  | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 5b9b1416d8f84..8a8221b428f13 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1914,7 +1914,10 @@ private void updateAllAsyncInternal0(
                             req.cleanup(!node.isLocal());
 
                         if (dhtFut != null) {
-                            if (req.writeSynchronizationMode() == PRIMARY_SYNC && !dhtFut.isDone()) {
+                            if (req.writeSynchronizationMode() == PRIMARY_SYNC
+                                // To avoid deadlock disable back-pressure for sender data node.
+                                && !ctx.discovery().cacheAffinityNode(ctx.discovery().node(nodeId), ctx.name())
+                                && !dhtFut.isDone()) {
                                 final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
 
                                 if (tracker != null && tracker instanceof GridNioMessageTracker) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
index 49e3e5ca07a15..30436f559f4ec 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
@@ -53,6 +54,8 @@ public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTe
         ccfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED);
         ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
 
+        ccfg.setNodeFilter(new NodeFilter());
+
         TestCommunicationSpi spi = new TestCommunicationSpi();
 
         spi.setMessageQueueLimit(100);
@@ -148,4 +151,17 @@ private static void sleep(long millis) {
             throw new IgniteSpiException(e);
         }
     }
+
+    /**
+     * Filters out server node producer.
+     */
+    private static class NodeFilter implements IgnitePredicate<ClusterNode> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode node) {
+            return !("server3".equals(node.attribute("org.apache.ignite.ignite.name")));
+        }
+    }
 }

From bb3ff120e6995431d10439243d8b163712de0e0e Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 10 Apr 2017 11:40:17 +0300
Subject: [PATCH 105/516] IGNITE-2466 - Disable back-pressure for sender data
 nodes to avoid deadlock.

(cherry picked from commit ba6227b)
---
 .../dht/atomic/GridDhtAtomicCache.java           |  5 ++++-
 .../CacheAtomicPrimarySyncBackPressureTest.java  | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 047be8751d7b1..4159359a2f780 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1909,7 +1909,10 @@ private void updateAllAsyncInternal0(
                             req.cleanup(!node.isLocal());
 
                         if (dhtFut != null) {
-                            if (req.writeSynchronizationMode() == PRIMARY_SYNC && !dhtFut.isDone()) {
+                            if (req.writeSynchronizationMode() == PRIMARY_SYNC
+                                // To avoid deadlock disable back-pressure for sender data node.
+                                && !ctx.discovery().cacheAffinityNode(ctx.discovery().node(nodeId), ctx.name())
+                                && !dhtFut.isDone()) {
                                 final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
 
                                 if (tracker != null && tracker instanceof GridNioMessageTracker) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
index 49e3e5ca07a15..30436f559f4ec 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
@@ -53,6 +54,8 @@ public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTe
         ccfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED);
         ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
 
+        ccfg.setNodeFilter(new NodeFilter());
+
         TestCommunicationSpi spi = new TestCommunicationSpi();
 
         spi.setMessageQueueLimit(100);
@@ -148,4 +151,17 @@ private static void sleep(long millis) {
             throw new IgniteSpiException(e);
         }
     }
+
+    /**
+     * Filters out server node producer.
+     */
+    private static class NodeFilter implements IgnitePredicate<ClusterNode> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode node) {
+            return !("server3".equals(node.attribute("org.apache.ignite.ignite.name")));
+        }
+    }
 }

From 960e454b4b68c76ed94ca58a3ee9a9d8e79c2039 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 10 Apr 2017 10:28:15 +0300
Subject: [PATCH 106/516] IGNITE-2466 - Use current NIO back pressure mechanism
 to limit received messages. Mark them process only when backups acknowledged.

(cherry picked from commit 220db88)
---
 .../managers/communication/GridIoManager.java |  12 +-
 .../dht/atomic/GridDhtAtomicCache.java        |  23 ++-
 .../util/nio/GridNioBackPressureControl.java  |  39 ++++-
 .../util/nio/GridNioMessageTracker.java       |   7 +
 ...acheAtomicPrimarySyncBackPressureTest.java | 151 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   3 +
 6 files changed, 220 insertions(+), 15 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 0b7c79006d627..ff7b8a2015dba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -736,7 +736,7 @@ private void processP2PMessage(
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     GridMessageListener lsnr = listenerGet0(msg.topic());
 
@@ -750,7 +750,7 @@ private void processP2PMessage(
                     invokeListener(msg.policy(), lsnr, nodeId, obj);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
 
                     msgC.run();
                 }
@@ -785,12 +785,12 @@ private void processRegularMessage(
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     processRegularMessage0(msg, nodeId);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
 
                     msgC.run();
                 }
@@ -1146,12 +1146,12 @@ else if (log.isDebugEnabled()) {
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     unwindMessageSet(msgSet0, lsnr);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
                 }
             }
         };
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 89bdf942c9a94..62a5506678b0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -84,6 +84,8 @@
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.nio.GridNioBackPressureControl;
+import org.apache.ignite.internal.util.nio.GridNioMessageTracker;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -92,14 +94,15 @@
 import org.apache.ignite.internal.util.typedef.CX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteOutClosure;
+import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
@@ -110,6 +113,7 @@
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
@@ -1904,8 +1908,23 @@ private void updateAllAsyncInternal0(
                         if (req.writeSynchronizationMode() != FULL_ASYNC)
                             req.cleanup(!node.isLocal());
 
-                        if (dhtFut != null)
+                        if (dhtFut != null) {
+                            if (req.writeSynchronizationMode() == PRIMARY_SYNC && !dhtFut.isDone()) {
+                                final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
+
+                                if (tracker != null && tracker instanceof GridNioMessageTracker) {
+                                    ((GridNioMessageTracker)tracker).onMessageReceived();
+
+                                    dhtFut.listen(new IgniteInClosure<IgniteInternalFuture<Void>>() {
+                                        @Override public void apply(IgniteInternalFuture<Void> fut) {
+                                            ((GridNioMessageTracker)tracker).onMessageProcessed();
+                                        }
+                                    });
+                                }
+                            }
+
                             ctx.mvcc().addAtomicFuture(dhtFut.version(), dhtFut);
+                        }
                     }
                     else
                         // Should remap all keys.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
index 96a1ab347d402..37d985f7ff100 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
@@ -17,14 +17,17 @@
 
 package org.apache.ignite.internal.util.nio;
 
+import org.apache.ignite.lang.IgniteRunnable;
+import org.jetbrains.annotations.Nullable;
+
 /**
  * Utility class that allows to ignore back-pressure control for threads that are processing messages.
  */
 public class GridNioBackPressureControl {
     /** Thread local flag indicating that thread is processing message. */
-    private static ThreadLocal<Boolean> threadProcMsg = new ThreadLocal<Boolean>() {
-        @Override protected Boolean initialValue() {
-            return Boolean.FALSE;
+    private static ThreadLocal<Holder> threadProcMsg = new ThreadLocal<Holder>() {
+        @Override protected Holder initialValue() {
+            return new Holder();
         }
     };
 
@@ -32,13 +35,35 @@ public class GridNioBackPressureControl {
      * @return Flag indicating whether current thread is processing message.
      */
     public static boolean threadProcessingMessage() {
-        return threadProcMsg.get();
+        return threadProcMsg.get().procMsg;
     }
 
     /**
      * @param processing Flag indicating whether current thread is processing message.
+     * @param tracker Thread local back pressure tracker of messages, associated with one connection.
+     */
+    public static void threadProcessingMessage(boolean processing, @Nullable IgniteRunnable tracker) {
+        Holder holder = threadProcMsg.get();
+
+        holder.procMsg = processing;
+        holder.tracker = tracker;
+    }
+
+    /**
+     * @return Thread local back pressure tracker of messages, associated with one connection.
      */
-    public static void threadProcessingMessage(boolean processing) {
-        threadProcMsg.set(processing);
+    @Nullable public static IgniteRunnable threadTracker() {
+        return threadProcMsg.get().tracker;
+    }
+
+    /**
+     *
+     */
+    private static class Holder {
+        /** Process message. */
+        private boolean procMsg;
+
+        /** Tracker. */
+        private IgniteRunnable tracker;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
index e02c7ca7fd6ee..f05ee0c60d045 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
@@ -92,6 +92,13 @@ public GridNioMessageTracker(GridNioSession ses, int msgQueueLimit) {
         }
     }
 
+    /**
+     *
+     */
+    public void onMessageProcessed() {
+        run();
+    }
+
     /**
      */
     public void onMessageReceived() {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
new file mode 100644
index 0000000000000..49e3e5ca07a15
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Checks that back-pressure control restricts uncontrolled growing
+ * of backup message queue. This means, if queue too big - any reads
+ * will be stopped until received acks from backup nodes.
+ */
+public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = new CacheConfiguration("cache");
+
+        ccfg.setBackups(1);
+
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
+        ccfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED);
+        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+        TestCommunicationSpi spi = new TestCommunicationSpi();
+
+        spi.setMessageQueueLimit(100);
+
+        cfg.setCommunicationSpi(spi);
+        cfg.setClientMode(gridName.contains("client"));
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientPut() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+
+        final Ignite client = startGrid("client");
+
+        checkBackPressure(client, srv1, srv2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testServerPut() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+
+        final Ignite client = startGrid("server3");
+
+        checkBackPressure(client, srv1, srv2);
+    }
+
+    /**
+     * @param client Producer node.
+     * @throws InterruptedException If failed.
+     */
+    private void checkBackPressure(Ignite client, final Ignite srv1, final Ignite srv2) throws Exception {
+        final IgniteCache<Integer, String> cache = client.cache("cache");
+
+        awaitPartitionMapExchange();
+
+        for (int i = 0; i < 10000; i++) {
+            cache.put(i, String.valueOf(i));
+
+            if (i % 100 == 0) {
+                int size1 = futuresNum(srv1);
+                int size2 = futuresNum(srv2);
+
+                assert size1 < 150 : size1;
+                assert size2 < 150 : size2;
+            }
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @return Size of the backup queue.
+     */
+    private int futuresNum(Ignite ignite) {
+        return ((IgniteKernal)ignite).context().cache().context().mvcc().atomicFutures().size();
+    }
+
+    /**
+     * Delays backup update acks.
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (((GridIoMessage)msg).message() instanceof GridDhtAtomicDeferredUpdateResponse)
+                sleep(100);
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     * @param millis Millis.
+     */
+    private static void sleep(long millis) {
+        try {
+            Thread.sleep(millis);
+        }
+        catch (InterruptedException e) {
+            throw new IgniteSpiException(e);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 5a09a1c5ae797..9fcf31a322f2a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -94,6 +94,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteSystemCacheOnClientTest;
 import org.apache.ignite.internal.processors.cache.MarshallerCacheJobRunNodeRestartTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheAffinityEarlyTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheGetFutureHangsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheNoValueClassOnServerNodeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheCreatePutMultiNodeSelfTest;
@@ -334,6 +335,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteCacheNearOnlyTxTest.class);
 
+        suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class);
+
         return suite;
     }
 }
\ No newline at end of file

From 207ccaa1ff5a021a25dc779b5b2d7bfdb0f3b647 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 10 Apr 2017 11:40:17 +0300
Subject: [PATCH 107/516] IGNITE-2466 - Disable back-pressure for sender data
 nodes to avoid deadlock.

(cherry picked from commit ba6227b)

(cherry picked from commit bb3ff12)
---
 .../dht/atomic/GridDhtAtomicCache.java           |  5 ++++-
 .../CacheAtomicPrimarySyncBackPressureTest.java  | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 62a5506678b0a..da16e7494e1e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1909,7 +1909,10 @@ private void updateAllAsyncInternal0(
                             req.cleanup(!node.isLocal());
 
                         if (dhtFut != null) {
-                            if (req.writeSynchronizationMode() == PRIMARY_SYNC && !dhtFut.isDone()) {
+                            if (req.writeSynchronizationMode() == PRIMARY_SYNC
+                                // To avoid deadlock disable back-pressure for sender data node.
+                                && !ctx.discovery().cacheAffinityNode(ctx.discovery().node(nodeId), ctx.name())
+                                && !dhtFut.isDone()) {
                                 final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
 
                                 if (tracker != null && tracker instanceof GridNioMessageTracker) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
index 49e3e5ca07a15..30436f559f4ec 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
@@ -53,6 +54,8 @@ public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTe
         ccfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED);
         ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
 
+        ccfg.setNodeFilter(new NodeFilter());
+
         TestCommunicationSpi spi = new TestCommunicationSpi();
 
         spi.setMessageQueueLimit(100);
@@ -148,4 +151,17 @@ private static void sleep(long millis) {
             throw new IgniteSpiException(e);
         }
     }
+
+    /**
+     * Filters out server node producer.
+     */
+    private static class NodeFilter implements IgnitePredicate<ClusterNode> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode node) {
+            return !("server3".equals(node.attribute("org.apache.ignite.ignite.name")));
+        }
+    }
 }

From 5fbbc37669bcf0557a98d8ab70c3dbb79ebcd340 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 5 Apr 2017 15:01:02 +0300
Subject: [PATCH 108/516] IGNITE-4917: Fixed failure when accessing
 BinaryObjectBuilder field value serialized with OptimizedMarshaller . This
 closes #1736.

(cherry picked from commit b8e3d1b)
---
 .../internal/binary/builder/BinaryBuilderReader.java     | 8 ++++++++
 .../binary/mutabletest/GridBinaryTestClasses.java        | 9 +++++++++
 2 files changed, 17 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
index baaabd6a1a0bd..02264e3311529 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
@@ -476,6 +476,14 @@ public Object getValueQuickly(int pos, int len) {
                 return new BinaryPlainBinaryObject(binaryObj);
             }
 
+            case GridBinaryMarshaller.OPTM_MARSH: {
+                final BinaryHeapInputStream bin = BinaryHeapInputStream.create(arr, pos + 1);
+
+                final Object obj = BinaryUtils.doReadOptimized(bin, ctx, U.resolveClassLoader(ctx.configuration()));
+
+                return obj;
+            }
+
             default:
                 throw new BinaryObjectException("Invalid flag value: " + type);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
index 5ddb87dcefb31..0d4de7fa0e215 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
@@ -24,6 +24,7 @@
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Date;
@@ -141,6 +142,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Integer i_;
 
+        /** */
+        public BigInteger bi_;
+
         /** */
         public Long l_;
 
@@ -150,6 +154,9 @@ public static class TestObjectAllTypes implements Serializable {
         /** */
         public Double d_;
 
+        /** */
+        public BigDecimal bd_;
+
         /** */
         public Character c_;
 
@@ -267,9 +274,11 @@ public void setDefaultData() {
             b_ = 11;
             s_ = 22;
             i_ = 33;
+            bi_ = new BigInteger("33000000000000");
             l_ = 44L;
             f_ = 55f;
             d_ = 66d;
+            bd_ = new BigDecimal("33000000000000.123456789");
             c_ = 'e';
             z_ = true;
 

From 315ff38eeef96f12954d6ff39c84d58b2b959667 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 6 Apr 2017 14:43:50 +0300
Subject: [PATCH 109/516] IGNITE-4879: Fixed System pool starvation while
 partition evicting.

---
 .../cache/distributed/dht/GridDhtLocalPartition.java          | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 3ce1dd83ae61e..a0ccc28c33b0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -294,8 +294,8 @@ void onRemoved(GridDhtCacheEntry entry) {
         // Make sure to remove exactly this entry.
         map.removeEntry(entry);
 
-        // Attempt to evict.
-        tryEvict();
+        // Attempt to evict in async way as multiple-threads can compete for same partition.
+        tryEvictAsync(true);
     }
 
     /**

From 89e9dbe484312c251f02c9fbe9698c3ac2e03df8 Mon Sep 17 00:00:00 2001
From: Alexander Fedotov <alexander.fedotoff@gmail.com>
Date: Mon, 10 Apr 2017 16:36:33 +0300
Subject: [PATCH 110/516] Fix
 org.apache.ignite.internal.processors.cache.expiry    
 .IgniteCacheExpiryPolicyAbstractTest#testNearExpiresWithCacheStore

---
 .../cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java      | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index 3339f65671c3e..41c915b774628 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -1061,6 +1061,9 @@ public void testNearExpiresOnClient() throws Exception {
      * @throws Exception If failed.
      */
     public void testNearExpiresWithCacheStore() throws Exception {
+        if(cacheMode() != PARTITIONED)
+            return;
+
         factory = CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 1));
 
         nearCache = true;

From 01ceeb13420b68edf12b0262fe0991e84c085dd8 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 6 Apr 2017 14:43:50 +0300
Subject: [PATCH 111/516] IGNITE-4863: Disallow change RootLogger log-level if
 it can have negative effect on other loggers. This closes #1687.

---
 .../ignite/logger/log4j/Log4JLogger.java      |  64 +++++-
 .../log4j/GridLog4jInitializationTest.java    | 212 ++++++++++++++++++
 .../log4j/GridLog4jInitializedTest.java       |  55 -----
 .../log4j/GridLog4jNotInitializedTest.java    |  46 ----
 .../testsuites/IgniteLog4jTestSuite.java      |   6 +-
 5 files changed, 268 insertions(+), 115 deletions(-)
 create mode 100644 modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializationTest.java
 delete mode 100644 modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializedTest.java
 delete mode 100644 modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jNotInitializedTest.java

diff --git a/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java b/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
index d5b0f0227bd85..f6ed83025af55 100644
--- a/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
+++ b/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
@@ -35,10 +35,12 @@
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.logger.LoggerNodeIdAware;
 import org.apache.log4j.Appender;
+import org.apache.log4j.AppenderSkeleton;
 import org.apache.log4j.Category;
 import org.apache.log4j.ConsoleAppender;
 import org.apache.log4j.FileAppender;
 import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 import org.apache.log4j.PatternLayout;
 import org.apache.log4j.varia.LevelRangeFilter;
@@ -78,6 +80,9 @@
  * injection.
  */
 public class Log4JLogger implements IgniteLogger, LoggerNodeIdAware, Log4jFileAware {
+    /** */
+    public static final String CONSOLE_ERR_APPENDER_NAME = "CONSOLE_ERR";
+
     /** Appenders. */
     private static Collection<FileAppender> fileAppenders = new GridConcurrentHashSet<>();
 
@@ -308,7 +313,7 @@ private void addConsoleAppenderIfNeeded(@Nullable Level logLevel,
                         Appender appender = (Appender)appenders.nextElement();
 
                         if (appender instanceof ConsoleAppender) {
-                            if ("CONSOLE_ERR".equals(appender.getName())) {
+                            if (CONSOLE_ERR_APPENDER_NAME.equals(appender.getName())) {
                                 // Treat CONSOLE_ERR appender as a system one and don't count it.
                                 errAppender = (ConsoleAppender)appender;
 
@@ -347,34 +352,62 @@ private void addConsoleAppenderIfNeeded(@Nullable Level logLevel,
                     if (errAppender.getThreshold() == Level.ERROR)
                         errAppender.setThreshold(Level.WARN);
                 }
-                else
-                    // No error console appender => create console appender with no level limit.
-                    rootCategory.addAppender(createConsoleAppender(Level.OFF));
+                else {
+                    // No error console appender => create console appender with.
+                    final AppenderSkeleton consoleAppender = createConsoleAppender(Level.OFF);
+
+                    consoleAppender.setThreshold(Level.INFO);
 
-                if (logLevel != null)
+                    rootCategory.addAppender(consoleAppender);
+                }
+
+                // Won't raise LogLevel if there is other loggers configured. As LogLevel can be inherited.
+                if (logLevel != null && !logLevel.isGreaterOrEqual(impl.getEffectiveLevel())) {
                     impl.setLevel(logLevel);
-            }
 
-            // If still don't have appenders, disable logging.
-            if (!isConfigured())
+                    impl.warn("RootLogger logging level has been dropped by Apache Ignite.\n"+
+                    "Set lower log level or configure ConsoleAppender manually or disable ConsoleAppender automatic creation.");
+                }
+            }
+            else if (!isConfigured() && !hasOtherLoggers()) {
+                // If still don't have appenders and other loggers configured, disable logging.
                 impl.setLevel(Level.OFF);
+            }
 
             quiet0 = quiet;
             inited = true;
         }
     }
 
+    /**
+     * Checks if there is other loggers configured.
+     *
+     * @return {@code True} if other logger found.
+     */
+    private boolean hasOtherLoggers() {
+        final Enumeration loggers = LogManager.getCurrentLoggers();
+
+        while (loggers.hasMoreElements()) {
+            Logger c = (Logger)loggers.nextElement();
+
+            if (c != impl && c.getAllAppenders().hasMoreElements())
+                return true;
+        }
+
+        return false;
+    }
+
     /**
      * Creates console appender with some reasonable default logging settings.
      *
      * @param maxLevel Max logging level.
      * @return New console appender.
      */
-    private Appender createConsoleAppender(Level maxLevel) {
+    private AppenderSkeleton createConsoleAppender(Level maxLevel) {
         String fmt = "[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n";
 
         // Configure output that should go to System.out
-        Appender app = new ConsoleAppender(new PatternLayout(fmt), ConsoleAppender.SYSTEM_OUT);
+        AppenderSkeleton app = new ConsoleAppender(new PatternLayout(fmt), ConsoleAppender.SYSTEM_OUT);
 
         LevelRangeFilter lvlFilter = new LevelRangeFilter();
 
@@ -532,4 +565,15 @@ public static Collection<String> logFiles() {
             }
         }
     }
+
+    /**
+     * For test purposes only.
+     */
+    static void reset(){
+        inited = false;
+
+        quiet0 = false;
+
+        fileAppenders.clear();
+    }
 }
diff --git a/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializationTest.java b/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializationTest.java
new file mode 100644
index 0000000000000..2a98490744db5
--- /dev/null
+++ b/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializationTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.logger.log4j;
+
+import junit.framework.TestCase;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+import org.apache.log4j.BasicConfigurator;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.varia.NullAppender;
+
+/**
+ * Log4j not initialized test.
+ */
+@GridCommonTest(group = "Logger")
+public class GridLog4jInitializationTest extends TestCase {
+    /** */
+    private static final boolean VERBOSE = true;
+
+    /** {@inheritDoc} */
+    @Override public void setUp() throws Exception {
+        super.setUp();
+
+        resetLogger();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void tearDown() throws Exception {
+        super.tearDown();
+
+        resetLogger();
+    }
+
+    /** */
+    private void resetLogger() {
+        Log4JLogger.reset();
+
+        LogManager.resetConfiguration();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_QUIET);
+    }
+
+    /** */
+    public void testLogNotInitialized() {
+        IgniteLogger log = new Log4JLogger().getLogger(GridLog4jInitializationTest.class);
+
+        if (VERBOSE)
+            printLoggerResults(log);
+
+        assertTrue(log instanceof Log4JLogger);
+
+        assertEquals(Level.OFF, LogManager.getRootLogger().getEffectiveLevel());
+    }
+
+    /** */
+    public void testLogInitialized() {
+        BasicConfigurator.configure();
+
+        IgniteLogger log = new Log4JLogger().getLogger(GridLog4jInitializationTest.class);
+
+        if (VERBOSE)
+            printLoggerResults(log);
+
+        assertTrue(log instanceof Log4JLogger);
+
+        assertEquals(Level.DEBUG, LogManager.getRootLogger().getEffectiveLevel());
+    }
+
+    /** */
+    public void testNoAppendersConfigured() {
+        LogManager.getRootLogger().setLevel(Level.WARN);
+
+        final Logger logger = LogManager.getLogger(GridLog4jInitializationTest.class);
+
+        assertEquals(Level.WARN, logger.getEffectiveLevel());
+
+        IgniteLogger log = new Log4JLogger().getLogger(GridLog4jInitializationTest.class);
+
+        if (VERBOSE)
+            printLoggerResults(log);
+
+        assertEquals(Level.OFF, logger.getEffectiveLevel());
+    }
+
+    /** */
+    public void testAutoAddConsoleAppender() {
+        System.setProperty(IgniteSystemProperties.IGNITE_QUIET, String.valueOf(false));
+
+        LogManager.getRootLogger().setLevel(Level.WARN);
+
+        final Logger logger = LogManager.getLogger(GridLog4jInitializationTest.class);
+
+        assertEquals(Level.WARN, logger.getEffectiveLevel());
+
+        IgniteLogger log = new Log4JLogger().getLogger(GridLog4jInitializationTest.class);
+
+        if (VERBOSE)
+            printLoggerResults(log);
+
+        assertEquals(Level.INFO, logger.getEffectiveLevel()); // LogLevel is allowed to be dropped.
+    }
+
+    /** */
+    public void testAutoAddConsoleAppender2() {
+        System.setProperty(IgniteSystemProperties.IGNITE_QUIET, String.valueOf(false));
+
+        LogManager.getRootLogger().setLevel(Level.DEBUG);
+
+        final Logger logger = LogManager.getLogger(GridLog4jInitializationTest.class);
+
+        assertEquals(Level.DEBUG, logger.getEffectiveLevel());
+
+        IgniteLogger log = new Log4JLogger().getLogger(GridLog4jInitializationTest.class);
+
+        if (VERBOSE)
+            printLoggerResults(log);
+
+        assertEquals(Level.DEBUG, logger.getEffectiveLevel()); // LogLevel should not change.
+    }
+
+    /** */
+    public void testOtherLoggerConfigured() {
+        LogManager.getRootLogger().setLevel(Level.DEBUG);
+
+        final Logger logger = LogManager.getLogger(GridLog4jInitializationTest.class);
+
+        logger.addAppender(new NullAppender());
+
+        assertEquals(Level.DEBUG, logger.getEffectiveLevel());
+
+        IgniteLogger log = new Log4JLogger().getLogger(GridLog4jInitializationTest.class);
+
+        if (VERBOSE)
+            printLoggerResults(log);
+
+        assertEquals(Level.DEBUG, logger.getEffectiveLevel()); // LogLevel should not be OFF.
+    }
+
+    /** */
+    public void testAutoAddConsoleAppenderWithOtherLoggerConfigured() {
+        System.setProperty(IgniteSystemProperties.IGNITE_QUIET, String.valueOf(false));
+
+        LogManager.getRootLogger().setLevel(Level.DEBUG);
+
+        final Logger logger = LogManager.getLogger(GridLog4jInitializationTest.class);
+
+        logger.addAppender(new NullAppender());
+
+        assertEquals(Level.DEBUG, logger.getEffectiveLevel());
+
+        IgniteLogger log = new Log4JLogger().getLogger(GridLog4jInitializationTest.class);
+
+        if (VERBOSE)
+            printLoggerResults(log);
+
+        assertEquals(Level.DEBUG, logger.getEffectiveLevel()); // LogLevel should not be raised.
+    }
+
+    /** */
+    public void testAutoAddConsoleAppenderWithOtherLoggerConfigured2() {
+        System.setProperty(IgniteSystemProperties.IGNITE_QUIET, String.valueOf(false));
+
+        LogManager.getRootLogger().setLevel(Level.WARN);
+
+        final Logger logger = LogManager.getLogger(GridLog4jInitializationTest.class);
+
+        logger.addAppender(new NullAppender());
+
+        assertEquals(Level.WARN, logger.getEffectiveLevel());
+
+        IgniteLogger log = new Log4JLogger().getLogger(GridLog4jInitializationTest.class);
+
+        if (VERBOSE)
+            printLoggerResults(log);
+
+        assertEquals(Level.INFO, logger.getEffectiveLevel()); // LogLevel is allowed to be dropped.
+    }
+
+    /** */
+    private void printLoggerResults(IgniteLogger log) {
+        if (log.isDebugEnabled())
+            log.debug("This is 'debug' message.");
+        else
+            System.out.println("DEBUG level is not enabled.");
+
+        if (log.isInfoEnabled())
+            log.info("This is 'info' message.");
+        else
+            System.out.println("INFO level is not enabled.");
+
+        log.warning("This is 'warning' message.");
+        log.error("This is 'error' message.");
+    }
+}
\ No newline at end of file
diff --git a/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializedTest.java b/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializedTest.java
deleted file mode 100644
index 94907f098b0c3..0000000000000
--- a/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializedTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.logger.log4j;
-
-import junit.framework.TestCase;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.testframework.junits.common.GridCommonTest;
-import org.apache.log4j.BasicConfigurator;
-
-/**
- * Log4j initialized test.
- */
-@GridCommonTest(group = "Logger")
-public class GridLog4jInitializedTest extends TestCase {
-
-    /**
-     * @throws Exception If failed.
-     */
-    @Override protected void setUp() throws Exception {
-        BasicConfigurator.configure();
-    }
-
-    /** */
-    public void testLogInitialize() {
-        IgniteLogger log = new Log4JLogger();
-
-        assert log.isInfoEnabled() == true;
-
-        if (log.isDebugEnabled())
-            log.debug("This is 'debug' message.");
-
-        log.info("This is 'info' message.");
-        log.warning("This is 'warning' message.");
-        log.warning("This is 'warning' message.", new Exception("It's a test warning exception"));
-        log.error("This is 'error' message.");
-        log.error("This is 'error' message.", new Exception("It's a test error exception"));
-
-        assert log.getLogger(GridLog4jInitializedTest.class.getName()) instanceof Log4JLogger;
-    }
-}
\ No newline at end of file
diff --git a/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jNotInitializedTest.java b/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jNotInitializedTest.java
deleted file mode 100644
index 390fdcb0a66bc..0000000000000
--- a/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jNotInitializedTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.logger.log4j;
-
-import junit.framework.TestCase;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.testframework.junits.common.GridCommonTest;
-
-/**
- * Log4j not initialized test.
- */
-@GridCommonTest(group = "Logger")
-public class GridLog4jNotInitializedTest extends TestCase {
-    /** */
-    public void testLogInitialize() {
-        IgniteLogger log = new Log4JLogger().getLogger(GridLog4jNotInitializedTest.class);
-
-        if (log.isDebugEnabled())
-            log.debug("This is 'debug' message.");
-        else
-            System.out.println("DEBUG level is not enabled.");
-
-        if (log.isInfoEnabled())
-            log.info("This is 'info' message.");
-        else
-            System.out.println("INFO level is not enabled.");
-
-        log.warning("This is 'warning' message.");
-        log.error("This is 'error' message.");
-    }
-}
\ No newline at end of file
diff --git a/modules/log4j/src/test/java/org/apache/ignite/testsuites/IgniteLog4jTestSuite.java b/modules/log4j/src/test/java/org/apache/ignite/testsuites/IgniteLog4jTestSuite.java
index f5f13d9e7ffb3..e20d32ce06c21 100644
--- a/modules/log4j/src/test/java/org/apache/ignite/testsuites/IgniteLog4jTestSuite.java
+++ b/modules/log4j/src/test/java/org/apache/ignite/testsuites/IgniteLog4jTestSuite.java
@@ -19,8 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.logger.log4j.GridLog4jCorrectFileNameTest;
-import org.apache.ignite.logger.log4j.GridLog4jInitializedTest;
-import org.apache.ignite.logger.log4j.GridLog4jNotInitializedTest;
+import org.apache.ignite.logger.log4j.GridLog4jInitializationTest;
 
 /**
  * Log4j logging tests.
@@ -33,8 +32,7 @@ public class IgniteLog4jTestSuite extends TestSuite {
     public static TestSuite suite() throws Exception {
         TestSuite suite = new TestSuite("Log4j Logging Test Suite");
 
-        suite.addTest(new TestSuite(GridLog4jInitializedTest.class));
-        suite.addTest(new TestSuite(GridLog4jNotInitializedTest.class));
+        suite.addTest(new TestSuite(GridLog4jInitializationTest.class));
         suite.addTest(new TestSuite(GridLog4jCorrectFileNameTest.class));
 
         return suite;

From 02b194268071b179d291b28472cef5d587e7558a Mon Sep 17 00:00:00 2001
From: Alexander Fedotov <alexander.fedotoff@gmail.com>
Date: Tue, 11 Apr 2017 12:00:59 +0300
Subject: [PATCH 112/516] Fix missing test resource directory for
 org.apache.ignite.spi.discovery.tcp    
 .TcpDiscoveryNodeAttributesUpdateOnReconnectTest.testReconnect

---
 modules/core/pom.xml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index f12cdfa3d510d..f786a2792523a 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -234,6 +234,9 @@
                     <exclude>**/*.java</exclude>
                 </excludes>
             </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+            </testResource>
         </testResources>
 
         <plugins>

From 20016a20f780eb3c21f249d3cb74d08018c4eea5 Mon Sep 17 00:00:00 2001
From: Alexander Fedotov <alexander.fedotoff@gmail.com>
Date: Tue, 11 Apr 2017 14:54:06 +0300
Subject: [PATCH 113/516] Fix
 org.apache.ignite.internal.processors.cache.expiry    
 .IgniteCacheExpiryPolicyAbstractTest#testNearExpiresWithCacheStore

---
 .../cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index 41c915b774628..e6737e43ce042 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -1087,9 +1087,11 @@ public void testNearExpiresWithCacheStore() throws Exception {
 
         cache.put(key, 1);
 
-        assertEquals(1, cache.localPeek(key, CachePeekMode.NEAR));
+        // Make entry cached in client NearCache.
         assertEquals(1, cache.get(key));
 
+        assertEquals(1, cache.localPeek(key, CachePeekMode.NEAR));
+
         waitExpired(key);
 
         for(int i = 0; i < gridCount(); i++)

From 6d12bd4b9fbded5e5862b5c3d89b05a5ddd11755 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 21 Mar 2017 17:54:51 +0300
Subject: [PATCH 114/516] IGNITE-4200: Added copying of the C++ binaries.

(cherry picked from commit 8b3860f)
---
 assembly/release-fabric-base.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml
index d7aa0a2ec2294..97222c94c635a 100644
--- a/assembly/release-fabric-base.xml
+++ b/assembly/release-fabric-base.xml
@@ -179,6 +179,12 @@
             <outputDirectory>/platforms/cpp/docs</outputDirectory>
         </fileSet>
 
+        <!-- Move CPP binaries. -->
+        <fileSet>
+            <directory>modules/platforms/cpp/bin</directory>
+            <outputDirectory>/platforms/cpp/bin</outputDirectory>
+        </fileSet>
+
         <!-- Other files. -->
         <fileSet>
             <directory>bin</directory>

From 0f7ef74216fab64f5d1d2b6d432b552b7fe40d2f Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 12 Apr 2017 13:01:25 +0300
Subject: [PATCH 115/516] IGNITE-4907: Fixed excessive service instances can be
 started with dynamic deployment. This closes #1766.

---
 .../service/GridServiceProcessor.java         |  2 +-
 ...rviceProcessorMultiNodeConfigSelfTest.java | 95 ++++++++++++++++---
 ...GridServiceProcessorMultiNodeSelfTest.java | 61 ++++++++++++
 3 files changed, 146 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index d0b2733473333..37bffc3130a73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -972,7 +972,7 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
                         int perNodeCnt = totalCnt != 0 ? totalCnt / size : maxPerNodeCnt;
                         int remainder = totalCnt != 0 ? totalCnt % size : 0;
 
-                        if (perNodeCnt > maxPerNodeCnt && maxPerNodeCnt != 0) {
+                        if (perNodeCnt >= maxPerNodeCnt && maxPerNodeCnt != 0) {
                             perNodeCnt = maxPerNodeCnt;
                             remainder = 0;
                         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index 1bd3b035a446c..9da62c0abb702 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -17,11 +17,13 @@
 
 package org.apache.ignite.internal.processors.service;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
-import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -38,6 +40,9 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
     /** Node singleton name. */
     private static final String NODE_SINGLE_BUT_CLIENT = "serviceConfigEachNodeButClient";
 
+    /** Node singleton name. */
+    private static final String NODE_SINGLE_WITH_LIMIT = "serviceConfigWithLimit";
+
     /** Affinity service name. */
     private static final String AFFINITY = "serviceConfigAffinity";
 
@@ -51,7 +56,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        ServiceConfiguration[] arr = new ServiceConfiguration[4];
+        List<ServiceConfiguration> cfgs = new ArrayList<>();
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -60,7 +65,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new DummyService());
 
-        arr[0] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -68,7 +73,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setMaxPerNodeCount(1);
         cfg.setService(new DummyService());
 
-        arr[1] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -79,7 +84,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new AffinityService(AFFINITY_KEY));
 
-        arr[2] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -88,9 +93,18 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
         cfg.setService(new DummyService());
 
-        arr[3] = cfg;
+        cfgs.add(cfg);
+
+        cfg = new ServiceConfiguration();
 
-        return arr;
+        cfg.setName(NODE_SINGLE_WITH_LIMIT);
+        cfg.setMaxPerNodeCount(1);
+        cfg.setTotalCount(nodeCount() + 1);
+        cfg.setService(new DummyService());
+
+        cfgs.add(cfg);
+
+        return cfgs.toArray(new ServiceConfiguration[cfgs.size()]);
     }
 
     /** {@inheritDoc} */
@@ -107,6 +121,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
                         DummyService.cancelled(NODE_SINGLE) == 0 &&
                         DummyService.started(NODE_SINGLE_BUT_CLIENT) == nodeCount() &&
                         DummyService.cancelled(NODE_SINGLE_BUT_CLIENT) == 0 &&
+                        DummyService.started(NODE_SINGLE_WITH_LIMIT) >= nodeCount() &&
+                        DummyService.cancelled(NODE_SINGLE_WITH_LIMIT) == 0 &&
                         actualCount(AFFINITY, randomGrid().services().serviceDescriptors()) == 1;
                 }
             },
@@ -170,6 +186,59 @@ public void testAffinityUpdateTopology() throws Exception {
         finally {
             stopExtraNodes(nodeCnt);
         }
+
+        checkCount(AFFINITY, g.services().serviceDescriptors(), 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployLimits() throws Exception {
+        final Ignite g = randomGrid();
+
+        final String name = NODE_SINGLE_WITH_LIMIT;
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
+
+        int extraNodes = 2;
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        DummyService.exeLatch(name, latch);
+
+        startExtraNodes(extraNodes);
+
+        try {
+            latch.await();
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount() + 1);
+        }
+        finally {
+            stopExtraNodes(extraNodes);
+        }
+
+        assertEquals(name, 1, DummyService.cancelled(name));
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
+    }
+
+    /**
+     * @param srvcName Service name
+     * @param expectedDeps Expected number of service deployments
+     *
+     */
+    private boolean waitForDeployment(final String srvcName, final int expectedDeps) throws IgniteInterruptedCheckedException {
+        final Ignite g = randomGrid();
+
+        return GridTestUtils.waitForCondition(new GridAbsPredicateX() {
+            @Override public boolean applyx() {
+                return actualCount(srvcName, g.services().serviceDescriptors())  == expectedDeps;
+            }
+        }, 1500);
     }
 
     /**
@@ -212,10 +281,6 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         try {
             latch.await();
 
-            // Ensure service is deployed.
-            assertNotNull(grid(nodeCount() + newNodes - 1).services()
-                .serviceProxy(NODE_SINGLE_BUT_CLIENT, Service.class, false, 2000));
-
             assertEquals(name, newNodes, DummyService.started(name));
             assertEquals(name, 0, DummyService.cancelled(name));
 
@@ -224,6 +289,10 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         finally {
             stopExtraNodes(newNodes);
         }
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 
     /**
@@ -253,5 +322,9 @@ private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Ex
         finally {
             stopExtraNodes(servers + clients);
         }
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index f7403dcc820ca..d133cf299a756 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -264,4 +264,65 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
             stopGrid("client");
         }
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployLimits() throws Exception {
+            String name = "serviceWithLimitsUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            final int totalInstances = nodeCount() + 1;
+
+            CountDownLatch latch = new CountDownLatch(nodeCount());
+
+            DummyService.exeLatch(name, latch);
+
+            ServiceConfiguration srvcCfg = new ServiceConfiguration();
+
+            srvcCfg.setName(name);
+            srvcCfg.setMaxPerNodeCount(1);
+            srvcCfg.setTotalCount(totalInstances);
+            srvcCfg.setService(new DummyService());
+
+            IgniteServices svcs = g.services().withAsync();
+
+            svcs.deploy(srvcCfg);
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
+            latch.await();
+
+            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
+            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount());
+
+            int extraNodes = 2;
+
+            latch = new CountDownLatch(1);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(2);
+
+            try {
+                latch.await();
+
+                TestCase.assertEquals(name, totalInstances, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), totalInstances);
+            }
+            finally {
+                stopExtraNodes(extraNodes);
+            }
+    }
 }
\ No newline at end of file

From 465084da5b00dcfc056d338f5d0a24875ca2af08 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 12 Apr 2017 13:01:25 +0300
Subject: [PATCH 116/516] IGNITE-4907: Fixed excessive service instances can be
 started with dynamic deployment. This closes #1766.

(cherry picked from commit 0f7ef74)
---
 .../service/GridServiceProcessor.java         |  2 +-
 ...rviceProcessorMultiNodeConfigSelfTest.java | 95 ++++++++++++++++---
 ...GridServiceProcessorMultiNodeSelfTest.java | 61 ++++++++++++
 3 files changed, 146 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index d0b2733473333..37bffc3130a73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -972,7 +972,7 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
                         int perNodeCnt = totalCnt != 0 ? totalCnt / size : maxPerNodeCnt;
                         int remainder = totalCnt != 0 ? totalCnt % size : 0;
 
-                        if (perNodeCnt > maxPerNodeCnt && maxPerNodeCnt != 0) {
+                        if (perNodeCnt >= maxPerNodeCnt && maxPerNodeCnt != 0) {
                             perNodeCnt = maxPerNodeCnt;
                             remainder = 0;
                         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index 1bd3b035a446c..9da62c0abb702 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -17,11 +17,13 @@
 
 package org.apache.ignite.internal.processors.service;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
-import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -38,6 +40,9 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
     /** Node singleton name. */
     private static final String NODE_SINGLE_BUT_CLIENT = "serviceConfigEachNodeButClient";
 
+    /** Node singleton name. */
+    private static final String NODE_SINGLE_WITH_LIMIT = "serviceConfigWithLimit";
+
     /** Affinity service name. */
     private static final String AFFINITY = "serviceConfigAffinity";
 
@@ -51,7 +56,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        ServiceConfiguration[] arr = new ServiceConfiguration[4];
+        List<ServiceConfiguration> cfgs = new ArrayList<>();
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -60,7 +65,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new DummyService());
 
-        arr[0] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -68,7 +73,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setMaxPerNodeCount(1);
         cfg.setService(new DummyService());
 
-        arr[1] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -79,7 +84,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new AffinityService(AFFINITY_KEY));
 
-        arr[2] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -88,9 +93,18 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
         cfg.setService(new DummyService());
 
-        arr[3] = cfg;
+        cfgs.add(cfg);
+
+        cfg = new ServiceConfiguration();
 
-        return arr;
+        cfg.setName(NODE_SINGLE_WITH_LIMIT);
+        cfg.setMaxPerNodeCount(1);
+        cfg.setTotalCount(nodeCount() + 1);
+        cfg.setService(new DummyService());
+
+        cfgs.add(cfg);
+
+        return cfgs.toArray(new ServiceConfiguration[cfgs.size()]);
     }
 
     /** {@inheritDoc} */
@@ -107,6 +121,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
                         DummyService.cancelled(NODE_SINGLE) == 0 &&
                         DummyService.started(NODE_SINGLE_BUT_CLIENT) == nodeCount() &&
                         DummyService.cancelled(NODE_SINGLE_BUT_CLIENT) == 0 &&
+                        DummyService.started(NODE_SINGLE_WITH_LIMIT) >= nodeCount() &&
+                        DummyService.cancelled(NODE_SINGLE_WITH_LIMIT) == 0 &&
                         actualCount(AFFINITY, randomGrid().services().serviceDescriptors()) == 1;
                 }
             },
@@ -170,6 +186,59 @@ public void testAffinityUpdateTopology() throws Exception {
         finally {
             stopExtraNodes(nodeCnt);
         }
+
+        checkCount(AFFINITY, g.services().serviceDescriptors(), 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployLimits() throws Exception {
+        final Ignite g = randomGrid();
+
+        final String name = NODE_SINGLE_WITH_LIMIT;
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
+
+        int extraNodes = 2;
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        DummyService.exeLatch(name, latch);
+
+        startExtraNodes(extraNodes);
+
+        try {
+            latch.await();
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount() + 1);
+        }
+        finally {
+            stopExtraNodes(extraNodes);
+        }
+
+        assertEquals(name, 1, DummyService.cancelled(name));
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
+    }
+
+    /**
+     * @param srvcName Service name
+     * @param expectedDeps Expected number of service deployments
+     *
+     */
+    private boolean waitForDeployment(final String srvcName, final int expectedDeps) throws IgniteInterruptedCheckedException {
+        final Ignite g = randomGrid();
+
+        return GridTestUtils.waitForCondition(new GridAbsPredicateX() {
+            @Override public boolean applyx() {
+                return actualCount(srvcName, g.services().serviceDescriptors())  == expectedDeps;
+            }
+        }, 1500);
     }
 
     /**
@@ -212,10 +281,6 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         try {
             latch.await();
 
-            // Ensure service is deployed.
-            assertNotNull(grid(nodeCount() + newNodes - 1).services()
-                .serviceProxy(NODE_SINGLE_BUT_CLIENT, Service.class, false, 2000));
-
             assertEquals(name, newNodes, DummyService.started(name));
             assertEquals(name, 0, DummyService.cancelled(name));
 
@@ -224,6 +289,10 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         finally {
             stopExtraNodes(newNodes);
         }
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 
     /**
@@ -253,5 +322,9 @@ private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Ex
         finally {
             stopExtraNodes(servers + clients);
         }
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index f7403dcc820ca..d133cf299a756 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -264,4 +264,65 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
             stopGrid("client");
         }
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployLimits() throws Exception {
+            String name = "serviceWithLimitsUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            final int totalInstances = nodeCount() + 1;
+
+            CountDownLatch latch = new CountDownLatch(nodeCount());
+
+            DummyService.exeLatch(name, latch);
+
+            ServiceConfiguration srvcCfg = new ServiceConfiguration();
+
+            srvcCfg.setName(name);
+            srvcCfg.setMaxPerNodeCount(1);
+            srvcCfg.setTotalCount(totalInstances);
+            srvcCfg.setService(new DummyService());
+
+            IgniteServices svcs = g.services().withAsync();
+
+            svcs.deploy(srvcCfg);
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
+            latch.await();
+
+            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
+            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount());
+
+            int extraNodes = 2;
+
+            latch = new CountDownLatch(1);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(2);
+
+            try {
+                latch.await();
+
+                TestCase.assertEquals(name, totalInstances, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), totalInstances);
+            }
+            finally {
+                stopExtraNodes(extraNodes);
+            }
+    }
 }
\ No newline at end of file

From a20c307df1dd000309a273ef93231fdc41a2a81c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 13 Apr 2017 09:31:17 +0300
Subject: [PATCH 117/516] IGNITE-4891 - Fix. Key is deserialized during
 transactional get() even if withKeepBinary is set

(Backport from master)
---
 .../transactions/IgniteTxLocalAdapter.java    |   5 +-
 .../cache/CacheKeepBinaryTransactionTest.java | 121 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |   2 +
 3 files changed, 126 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index ea816fe6ce5f0..7f161859da101 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1823,8 +1823,9 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                         for (Iterator<KeyCacheObject> it = missed.keySet().iterator(); it.hasNext(); ) {
                             KeyCacheObject cacheKey = it.next();
 
-                            K keyVal =
-                                (K)(keepCacheObjects ? cacheKey : cacheKey.value(cacheCtx.cacheObjectContext(), false));
+                            K keyVal = (K)(keepCacheObjects ? cacheKey
+                                : cacheCtx.cacheObjectContext()
+                                    .unwrapBinaryIfNeeded(cacheKey, !deserializeBinary, false));
 
                             if (retMap.containsKey(keyVal))
                                 it.remove();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java
new file mode 100644
index 0000000000000..27d7c7f004330
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteBinary;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.TransactionConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+/**
+ * Test that no deserialization happens with binary objects and keepBinary set flag.
+ */
+public class CacheKeepBinaryTransactionTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TransactionConfiguration txCfg = new TransactionConfiguration();
+        txCfg.setDefaultTxConcurrency(TransactionConcurrency.OPTIMISTIC);
+        txCfg.setDefaultTxIsolation(TransactionIsolation.REPEATABLE_READ);
+
+        cfg.setTransactionConfiguration(txCfg);
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        CacheConfiguration ccfg = new CacheConfiguration("tx-cache");
+        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid(0);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBinaryGet() throws Exception {
+        IgniteEx ignite = grid(0);
+        IgniteCache<Object, Object> cache = ignite.cache("tx-cache").withKeepBinary();
+
+        try (Transaction tx = ignite.transactions().txStart()) {
+            BinaryObject key = ignite.binary().builder("test1")
+                .setField("id", 1).hashCode(1).build();
+
+            assertNull(cache.get(key));
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBinaryContains() throws Exception {
+        IgniteEx ignite = grid(0);
+        IgniteCache<Object, Object> cache = ignite.cache("tx-cache").withKeepBinary();
+
+        try (Transaction tx = ignite.transactions().txStart()) {
+            BinaryObject key = ignite.binary().builder("test2")
+                .setField("id", 1).hashCode(1).build();
+
+            assertFalse(cache.containsKey(key));
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBinaryPutGetContains() throws Exception {
+        IgniteEx ignite = grid(0);
+        IgniteCache<Object, Object> cache = ignite.cache("tx-cache").withKeepBinary();
+
+        try (Transaction tx = ignite.transactions().txStart()) {
+            IgniteBinary binary = ignite.binary();
+
+            BinaryObject key = binary.builder("test-key").setField("id", 1).hashCode(1).build();
+            BinaryObject val = binary.builder("test-val").setField("id", 22).build();
+
+            cache.put(key, val);
+
+            assertTrue(cache.containsKey(key));
+            assertEquals(val, cache.get(key));
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 7f0e23c29aa19..421676654e7d5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.CacheKeepBinaryTransactionTest;
 import org.apache.ignite.internal.processors.cache.CacheNearReaderUpdateTest;
 import org.apache.ignite.internal.processors.cache.CacheSerializableTransactionsTest;
 import org.apache.ignite.internal.processors.cache.GridCacheSwapSpaceSpiConsistencySelfTest;
@@ -50,6 +51,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheWriteBehindNoUpdateSelfTest.class);
         suite.addTestSuite(IgniteCachePutStackOverflowSelfTest.class);
         suite.addTestSuite(GridCacheSwapSpaceSpiConsistencySelfTest.class);
+        suite.addTestSuite(CacheKeepBinaryTransactionTest.class);
 
         suite.addTestSuite(CacheLateAffinityAssignmentTest.class);
         suite.addTestSuite(CacheLateAffinityAssignmentFairAffinityTest.class);

From 630558dfeb373f237057e394e8f2f63230d59dab Mon Sep 17 00:00:00 2001
From: vladisav <vladisavj@gmail.com>
Date: Thu, 13 Apr 2017 13:24:42 +0300
Subject: [PATCH 118/516] ignite-4173 IgniteSemaphore with failoverSafe enabled
 doesn't release permits in case permits owner node left topology

Backport from master.

(cherry picked from commit 76485fc)
---
 .../datastructures/GridCacheSemaphoreImpl.java       | 12 ++++++++++++
 .../SemaphoreFailoverSafeReleasePermitsTest.java     |  9 ++++-----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
index a11c79db6282c..e3cfd778da4e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
@@ -532,6 +532,18 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public void onNodeRemoved(UUID nodeId) {
+        try {
+            initializeSemaphore();
+        }
+        catch (IgniteCheckedException e) {
+            U.error(log, "Failed to recover from failover because distributed semaphore cannot be initialized " +
+                "(Ignore if this node is failing also)." );
+
+            // Degrade gracefully, no exception is thrown
+            // because other semaphores might also attempt to recover from failover.
+            return;
+        }
+
         int numPermits = sync.getPermitsForNode(nodeId);
 
         if (numPermits > 0) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/SemaphoreFailoverSafeReleasePermitsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/SemaphoreFailoverSafeReleasePermitsTest.java
index 241253d3de8bd..9a588e7c419f7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/SemaphoreFailoverSafeReleasePermitsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/SemaphoreFailoverSafeReleasePermitsTest.java
@@ -92,6 +92,9 @@ private void doTest() throws Exception {
 
             IgniteSemaphore sem = ignite.semaphore("sem", 1, true, true);
 
+            // Initialize second semaphore before the first one is broken.
+            IgniteSemaphore sem2 = grid(1).semaphore("sem", 1, true, true);
+
             assertEquals(1, sem.availablePermits());
 
             sem.acquire(1);
@@ -102,11 +105,7 @@ private void doTest() throws Exception {
 
             awaitPartitionMapExchange();
 
-            ignite = grid(1);
-
-            sem = ignite.semaphore("sem", 1, true, true);
-
-            assertTrue(sem.tryAcquire(1, 5000, TimeUnit.MILLISECONDS));
+            assertTrue(sem2.tryAcquire(1, 5000, TimeUnit.MILLISECONDS));
         }
         finally {
             stopAllGrids();

From 870b752c095ed3776e91a65b99763142b9f2ebc0 Mon Sep 17 00:00:00 2001
From: Vladisav Jelisavcic <vladisavj@gmail.com>
Date: Tue, 11 Apr 2017 14:09:12 +0300
Subject: [PATCH 119/516] ignite-1977 - fixed IgniteSemaphore fault tolerance.

Backport from master.

(cherry picked from commit 902bf42)
---
 .../GridCacheSemaphoreImpl.java               | 74 ++++++++++++++++---
 .../GridCacheSemaphoreState.java              | 22 ++++++
 ...bstractDataStructuresFailoverSelfTest.java | 21 ++++--
 3 files changed, 102 insertions(+), 15 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
index e3cfd778da4e1..c7426300db5e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java
@@ -232,6 +232,10 @@ final int nonfairTryAcquireShared(int acquires) {
 
         /** {@inheritDoc} */
         @Override protected final boolean tryReleaseShared(int releases) {
+            // Fail-fast path.
+            if(broken)
+                return true;
+
             // Check if some other node updated the state.
             // This method is called with release==0 only when trying to wake through update.
             if (releases == 0)
@@ -295,6 +299,13 @@ protected boolean compareAndSetGlobalState(final int expVal, final int newVal, f
                                     throw new IgniteCheckedException("Failed to find semaphore with given name: " +
                                         name);
 
+                                // Abort if state is already broken.
+                                if (val.isBroken()) {
+                                    tx.rollback();
+
+                                    return true;
+                                }
+
                                 boolean retVal = val.getCount() == expVal;
 
                                 if (retVal) {
@@ -349,11 +360,13 @@ protected boolean compareAndSetGlobalState(final int expVal, final int newVal, f
 
         /**
          * This method is used for releasing the permits acquired by failing node.
+         * In case the semaphore is broken, no permits are released and semaphore is set (globally) to broken state.
          *
          * @param nodeId ID of the failing node.
+         * @param broken Flag indicating that this semaphore is broken.
          * @return True if this is the call that succeeded to change the global state.
          */
-        protected boolean releaseFailedNode(final UUID nodeId) {
+        protected boolean releaseFailedNode(final UUID nodeId, final boolean broken) {
             try {
                 return CU.outTx(
                     retryTopologySafe(new Callable<Boolean>() {
@@ -369,6 +382,25 @@ protected boolean releaseFailedNode(final UUID nodeId) {
                                     throw new IgniteCheckedException("Failed to find semaphore with given name: " +
                                         name);
 
+                                // Quit early if semaphore is already broken.
+                                if( val.isBroken()) {
+                                    tx.rollback();
+
+                                    return false;
+                                }
+
+                                // Mark semaphore as broken. No permits are released,
+                                // since semaphore is useless from now on.
+                                if (broken) {
+                                    val.setBroken(true);
+
+                                    semView.put(key, val);
+
+                                    tx.commit();
+
+                                    return true;
+                                }
+
                                 Map<UUID, Integer> map = val.getWaiters();
 
                                 if (!map.containsKey(nodeId)) {
@@ -473,7 +505,11 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
                                 tx.commit();
 
-                                return new Sync(count, waiters, failoverSafe);
+                                Sync sync = new Sync(count, waiters, failoverSafe);
+
+                                sync.setBroken(val.isBroken());
+
+                                return sync;
                             }
                         }
                     }),
@@ -520,6 +556,9 @@ private void initializeSemaphore() throws IgniteCheckedException {
         if (sync == null)
             return;
 
+        // Update broken flag.
+        sync.setBroken(val.isBroken());
+
         // Update permission count.
         sync.setPermits(val.getCount());
 
@@ -547,10 +586,13 @@ private void initializeSemaphore() throws IgniteCheckedException {
         int numPermits = sync.getPermitsForNode(nodeId);
 
         if (numPermits > 0) {
-            if (sync.failoverSafe)
-                // Release permits acquired by threads on failing node.
-                sync.releaseFailedNode(nodeId);
-            else {
+            // Semaphore is broken if reaches this point in non-failover safe mode.
+            boolean broken = !sync.failoverSafe;
+
+            // Release permits acquired by threads on failing node.
+            sync.releaseFailedNode(nodeId, broken);
+
+            if (broken) {
                 // Interrupt every waiting thread if this semaphore is not failover safe.
                 sync.setBroken(true);
 
@@ -626,8 +668,11 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
             sync.acquireSharedInterruptibly(permits);
 
-            if (isBroken())
+            if (isBroken()) {
+                Thread.interrupted(); // Clear interrupt flag.
+
                 throw new InterruptedException();
+            }
         }
         catch (IgniteCheckedException e) {
             throw U.convertException(e);
@@ -743,8 +788,11 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
             boolean result = sync.nonfairTryAcquireShared(1) >= 0;
 
-            if (isBroken())
+            if (isBroken()) {
+                Thread.interrupted(); // Clear interrupt flag.
+
                 throw new InterruptedException();
+            }
 
             return result;
         }
@@ -768,8 +816,11 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
             boolean result = sync.tryAcquireSharedNanos(1, unit.toNanos(timeout));
 
-            if (isBroken())
+            if (isBroken()) {
+                Thread.interrupted(); // Clear interrupt flag.
+
                 throw new InterruptedException();
+            }
 
             return result;
         }
@@ -837,8 +888,11 @@ private void initializeSemaphore() throws IgniteCheckedException {
 
             boolean result = sync.tryAcquireSharedNanos(permits, unit.toNanos(timeout));
 
-            if (isBroken())
+            if (isBroken()) {
+                Thread.interrupted();
+
                 throw new InterruptedException();
+            }
 
             return result;
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreState.java
index 50cdf102e0331..cdff9c51d7a35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreState.java
@@ -46,6 +46,9 @@ public class GridCacheSemaphoreState implements GridCacheInternal, Externalizabl
     /** FailoverSafe flag. */
     private boolean failoverSafe;
 
+    /** Flag indicating that semaphore is no longer safe to use. */
+    private boolean broken;
+
     /**
      * Constructor.
      *
@@ -101,6 +104,21 @@ public boolean isFailoverSafe() {
         return failoverSafe;
     }
 
+    /**
+     * @return broken flag.
+     */
+    public boolean isBroken() {
+        return broken;
+    }
+
+    /**
+     *
+     * @param broken Flag indicating that this semaphore should be no longer used.
+     */
+    public void setBroken(boolean broken) {
+        this.broken = broken;
+    }
+
     /** {@inheritDoc} */
     @Override public Object clone() throws CloneNotSupportedException {
         return super.clone();
@@ -120,6 +138,8 @@ public boolean isFailoverSafe() {
                 out.writeInt(e.getValue());
             }
         }
+
+        out.writeBoolean(broken);
     }
 
     /** {@inheritDoc} */
@@ -135,6 +155,8 @@ public boolean isFailoverSafe() {
             for (int i = 0; i < size; i++)
                 waiters.put(U.readUuid(in), in.readInt());
         }
+
+        broken = in.readBoolean();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
index 6ba65abf5dec9..9295770796381 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
@@ -529,8 +529,6 @@ public void testSemaphoreMultipleTopologyChangeNonFailoverSafe() throws Exceptio
      * @throws Exception If failed.
      */
     private void doTestSemaphore(ConstantTopologyChangeWorker topWorker, final boolean failoverSafe) throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-1977");
-
         final int permits = topWorker instanceof MultipleTopologyChangeWorker ||
             topWorker instanceof PartitionedMultipleTopologyChangeWorker ? TOP_CHANGE_THREAD_CNT * 3 :
             TOP_CHANGE_CNT;
@@ -547,9 +545,14 @@ private void doTestSemaphore(ConstantTopologyChangeWorker topWorker, final boole
                             break;
                         }
                         catch (IgniteInterruptedException e) {
-                            // Exception may happen in non failover safe mode.
+                           // Exception may happen in non failover safe mode.
                             if (failoverSafe)
                                 throw e;
+                            else {
+                                // In non-failoverSafe mode semaphore is not safe to be reused,
+                                // and should always be discarded after exception is caught.
+                                break;
+                            }
                         }
                     }
 
@@ -568,6 +571,11 @@ private void doTestSemaphore(ConstantTopologyChangeWorker topWorker, final boole
                         // Exception may happen in non failover safe mode.
                         if (failoverSafe)
                             throw e;
+                        else {
+                            // In non-failoverSafe mode semaphore is not safe to be reused,
+                            // and should always be discarded after exception is caught.
+                            break;
+                        }
                     }
                 }
 
@@ -580,8 +588,11 @@ private void doTestSemaphore(ConstantTopologyChangeWorker topWorker, final boole
 
             fut.get();
 
-            for (Ignite g : G.allGrids())
-                assertEquals(permits, g.semaphore(STRUCTURE_NAME, permits, false, false).availablePermits());
+            // Semaphore is left in proper state only if failoverSafe mode is used.
+            if (failoverSafe) {
+                for (Ignite g : G.allGrids())
+                    assertEquals(permits, g.semaphore(STRUCTURE_NAME, permits, false, false).availablePermits());
+            }
         }
     }
 

From cd0b92950c6691c6fc1a26cb4f7e55f5ee459298 Mon Sep 17 00:00:00 2001
From: Yakov Zhdanov <yzhdanov@gridgain.com>
Date: Thu, 13 Apr 2017 15:52:20 +0300
Subject: [PATCH 120/516] ignite-4946 GridCacheP2PUndeploySelfTest became
 failed

(cherry picked from commit d298e75)
---
 .../eventstorage/GridEventStorageManager.java | 341 ++++++++++--------
 1 file changed, 187 insertions(+), 154 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
index a2c64ba33d8b3..406da2d284b63 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
@@ -21,6 +21,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EventListener;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
@@ -33,6 +34,7 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteEvents;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.DiscoveryEvent;
@@ -78,10 +80,7 @@
  */
 public class GridEventStorageManager extends GridManagerAdapter<EventStorageSpi> {
     /** Local event listeners. */
-    private final ConcurrentMap<Integer, Set<GridLocalEventListener>> lsnrs = new ConcurrentHashMap8<>();
-
-    /** Internal discovery listeners. */
-    private final ConcurrentMap<Integer, Set<DiscoveryEventListener>> discoLsnrs = new ConcurrentHashMap8<>();
+    private final ConcurrentMap<Integer, Set<EventListener>> lsnrs = new ConcurrentHashMap8<>();
 
     /** Busy lock to control activity of threads. */
     private final ReadWriteLock busyLock = new ReentrantReadWriteLock();
@@ -200,7 +199,7 @@ public GridEventStorageManager(GridKernalContext ctx) {
     @Override public void printMemoryStats() {
         int lsnrsCnt = 0;
 
-        for (Set<GridLocalEventListener> lsnrs0 : lsnrs.values())
+        for (Set<EventListener> lsnrs0 : lsnrs.values())
             lsnrsCnt += lsnrs0.size();
 
         X.println(">>>");
@@ -238,7 +237,6 @@ private void leaveBusy() {
         msgLsnr = null;
 
         lsnrs.clear();
-        discoLsnrs.clear();
     }
 
     /** {@inheritDoc} */
@@ -274,6 +272,26 @@ private void leaveBusy() {
      * @param evt Event to record.
      */
     public void record(Event evt) {
+        record0(evt);
+    }
+
+    /**
+     * Records discovery events.
+     *
+     * @param evt Event to record.
+     * @param discoCache Discovery cache.
+     */
+    public void record(DiscoveryEvent evt, DiscoCache discoCache) {
+        record0(evt, discoCache);
+    }
+
+    /**
+     * Records event if it's recordable.
+     *
+     * @param evt Event to record.
+     * @param params Additional parameters.
+     */
+    private void record0(Event evt, Object... params) {
         assert evt != null;
 
         if (!enterBusy())
@@ -297,31 +315,7 @@ public void record(Event evt) {
                 }
 
             if (isRecordable(type))
-                notifyListeners(evt);
-        }
-        finally {
-            leaveBusy();
-        }
-    }
-
-    /**
-     * Records discovery events.
-     *
-     * @param evt Event to record.
-     * @param discoCache Discovery cache.
-     */
-    public void record(DiscoveryEvent evt, DiscoCache discoCache) {
-        assert evt != null;
-
-        if (!enterBusy())
-            return;
-
-        try {
-            // Notify internal discovery listeners first.
-            notifyDiscoveryListeners(evt, discoCache);
-
-            // Notify all other registered listeners.
-            record(evt);
+                notifyListeners(lsnrs.get(evt.type()), evt, params);
         }
         finally {
             leaveBusy();
@@ -571,6 +565,8 @@ private boolean binarySearch(@Nullable int[] arr, int val) {
      * @param types Event types to subscribe listener for.
      */
     public void addLocalEventListener(IgnitePredicate<? extends Event> lsnr, int[] types) {
+        assert lsnr != null;
+
         try {
             ctx.resource().injectGeneric(lsnr);
         }
@@ -578,7 +574,7 @@ public void addLocalEventListener(IgnitePredicate<? extends Event> lsnr, int[] t
             throw new IgniteException("Failed to inject resources to event listener: " + lsnr, e);
         }
 
-        addLocalEventListener(new UserListenerWrapper(lsnr), types);
+        addEventListener(new UserListenerWrapper(lsnr), types);
     }
 
     /**
@@ -594,20 +590,7 @@ public void addLocalEventListener(GridLocalEventListener lsnr, int[] types) {
         assert types != null;
         assert types.length > 0;
 
-        if (!enterBusy())
-            return;
-
-        try {
-            for (int t : types) {
-                getOrCreate(lsnrs, t).add(lsnr);
-
-                if (!isRecordable(t))
-                    U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
-            }
-        }
-        finally {
-            leaveBusy();
-        }
+        addEventListener(new LocalListenerWrapper(lsnr), types);
     }
 
     /**
@@ -620,27 +603,7 @@ public void addLocalEventListener(GridLocalEventListener lsnr, int[] types) {
     public void addLocalEventListener(GridLocalEventListener lsnr, int type, @Nullable int... types) {
         assert lsnr != null;
 
-        if (!enterBusy())
-            return;
-
-        try {
-            getOrCreate(lsnrs, type).add(lsnr);
-
-            if (!isRecordable(type))
-                U.warn(log, "Added listener for disabled event type: " + U.gridEventName(type));
-
-            if (types != null) {
-                for (int t : types) {
-                    getOrCreate(lsnrs, t).add(lsnr);
-
-                    if (!isRecordable(t))
-                        U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
-                }
-            }
-        }
-        finally {
-            leaveBusy();
-        }
+        addEventListener(new LocalListenerWrapper(lsnr), type, types);
     }
 
     /**
@@ -656,12 +619,40 @@ public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int[] types)
         assert types != null;
         assert types.length > 0;
 
+        addEventListener(new DiscoveryListenerWrapper(lsnr), types);
+    }
+
+    /**
+     * Adds discovery event listener.
+     *
+     * @param lsnr Listener to add.
+     * @param type Event type to subscribe listener for.
+     * @param types Additional event types to subscribe listener for.
+     */
+    public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int type, @Nullable int... types) {
+        assert lsnr != null;
+
+        addEventListener(new DiscoveryListenerWrapper(lsnr), type, types);
+    }
+
+    /**
+     * Adds local event listener. Note that this method specifically disallow an empty
+     * array of event type to prevent accidental subscription for all system event that
+     * may lead to a drastic performance decrease.
+     *
+     * @param lsnr Listener to add.
+     * @param types Event types to subscribe listener for.
+     */
+    private void addEventListener(EventListener lsnr, int[] types) {
         if (!enterBusy())
             return;
 
         try {
             for (int t : types) {
-                getOrCreate(discoLsnrs, t).add(lsnr);
+                getOrCreate(lsnrs, t).add(lsnr);
+
+                if (!isRecordable(t))
+                    U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
             }
         }
         finally {
@@ -670,24 +661,28 @@ public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int[] types)
     }
 
     /**
-     * Adds discovery event listener.
+     * Adds local event listener.
      *
      * @param lsnr Listener to add.
      * @param type Event type to subscribe listener for.
      * @param types Additional event types to subscribe listener for.
      */
-    public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int type, @Nullable int... types) {
-        assert lsnr != null;
-
+    private void addEventListener(EventListener lsnr, int type, @Nullable int... types) {
         if (!enterBusy())
             return;
 
         try {
-            getOrCreate(discoLsnrs, type).add(lsnr);
+            getOrCreate(lsnrs, type).add(lsnr);
+
+            if (!isRecordable(type))
+                U.warn(log, "Added listener for disabled event type: " + U.gridEventName(type));
 
             if (types != null) {
                 for (int t : types) {
-                    getOrCreate(discoLsnrs, t).add(lsnr);
+                    getOrCreate(lsnrs, t).add(lsnr);
+
+                    if (!isRecordable(t))
+                        U.warn(log, "Added listener for disabled event type: " + U.gridEventName(t));
                 }
             }
         }
@@ -696,6 +691,7 @@ public void addDiscoveryEventListener(DiscoveryEventListener lsnr, int type, @Nu
         }
     }
 
+
     /**
      * @param lsnrs Listeners map.
      * @param type Event type.
@@ -727,7 +723,9 @@ private <T> Collection<T> getOrCreate(ConcurrentMap<Integer, Set<T>> lsnrs, Inte
      * @return Returns {@code true} if removed.
      */
     public boolean removeLocalEventListener(IgnitePredicate<? extends Event> lsnr, @Nullable int... types) {
-        return removeLocalEventListener(new UserListenerWrapper(lsnr), types);
+        assert lsnr != null;
+
+        return removeEventListener(new UserListenerWrapper(lsnr), types);
     }
 
     /**
@@ -741,33 +739,21 @@ public boolean removeLocalEventListener(IgnitePredicate<? extends Event> lsnr, @
     public boolean removeLocalEventListener(GridLocalEventListener lsnr, @Nullable int... types) {
         assert lsnr != null;
 
-        boolean found = false;
-
-        if (F.isEmpty(types)) {
-            for (Set<GridLocalEventListener> set : lsnrs.values())
-                if (set.remove(lsnr))
-                    found = true;
-        }
-        else {
-            assert types != null;
-
-            for (int type : types) {
-                Set<GridLocalEventListener> set = lsnrs.get(type);
-
-                if (set != null && set.remove(lsnr))
-                    found = true;
-            }
-        }
-
-        if (lsnr instanceof UserListenerWrapper)
-        {
-            IgnitePredicate p = ((UserListenerWrapper)lsnr).listener();
+        return removeEventListener(new LocalListenerWrapper(lsnr), types);
+    }
 
-            if (p instanceof PlatformEventFilterListener)
-                ((PlatformEventFilterListener)p).onClose();
-        }
+    /**
+     * Removes listener for specified events, if any. If no event types provided - it
+     * remove the listener for all its registered events.
+     *
+     * @param lsnr Listener.
+     * @param types Event types.
+     * @return Returns {@code true} if removed.
+     */
+    public boolean removeDiscoveryEventListener(DiscoveryEventListener lsnr, @Nullable int... types) {
+        assert lsnr != null;
 
-        return found;
+        return removeEventListener(new DiscoveryListenerWrapper(lsnr), types);
     }
 
     /**
@@ -778,13 +764,13 @@ public boolean removeLocalEventListener(GridLocalEventListener lsnr, @Nullable i
      * @param types Event types.
      * @return Returns {@code true} if removed.
      */
-    public boolean removeDiscoveryEventListener(DiscoveryEventListener lsnr, @Nullable int... types) {
+    private boolean removeEventListener(EventListener lsnr, @Nullable int[] types) {
         assert lsnr != null;
 
         boolean found = false;
 
         if (F.isEmpty(types)) {
-            for (Set<DiscoveryEventListener> set : discoLsnrs.values())
+            for (Set<EventListener> set : lsnrs.values())
                 if (set.remove(lsnr))
                     found = true;
         }
@@ -792,13 +778,21 @@ public boolean removeDiscoveryEventListener(DiscoveryEventListener lsnr, @Nullab
             assert types != null;
 
             for (int type : types) {
-                Set<DiscoveryEventListener> set = discoLsnrs.get(type);
+                Set<EventListener> set = lsnrs.get(type);
 
                 if (set != null && set.remove(lsnr))
                     found = true;
             }
         }
 
+        if (lsnr instanceof UserListenerWrapper)
+        {
+            IgnitePredicate p = ((UserListenerWrapper)lsnr).listener();
+
+            if (p instanceof PlatformEventFilterListener)
+                ((PlatformEventFilterListener)p).onClose();
+        }
+
         return found;
     }
 
@@ -861,63 +855,19 @@ public Event waitForEvent(long timeout, @Nullable Runnable c,
         return fut.get(timeout);
     }
 
-    /**
-     * @param evt Event to notify about.
-     */
-    private void notifyListeners(Event evt) {
-        assert evt != null;
-
-        notifyListeners(lsnrs.get(evt.type()), evt);
-    }
-
     /**
      * @param set Set of listeners.
      * @param evt Grid event.
      */
-    private void notifyListeners(@Nullable Collection<GridLocalEventListener> set, Event evt) {
-        assert evt != null;
-
-        if (!F.isEmpty(set)) {
-            assert set != null;
-
-            for (GridLocalEventListener lsnr : set) {
-                try {
-                    lsnr.onEvent(evt);
-                }
-                catch (Throwable e) {
-                    U.error(log, "Unexpected exception in listener notification for event: " + evt, e);
-
-                    if (e instanceof Error)
-                        throw (Error)e;
-                }
-            }
-        }
-    }
-
-    /**
-     * @param evt Discovery event
-     * @param cache Discovery cache.
-     */
-    private void notifyDiscoveryListeners(DiscoveryEvent evt, DiscoCache cache) {
-        assert evt != null;
-
-        notifyDiscoveryListeners(discoLsnrs.get(evt.type()), evt, cache);
-    }
-
-    /**
-     * @param set Set of listeners.
-     * @param evt Discovery event.
-     * @param cache Discovery cache.
-     */
-    private void notifyDiscoveryListeners(@Nullable Collection<DiscoveryEventListener> set, DiscoveryEvent evt, DiscoCache cache) {
+    private void notifyListeners(@Nullable Collection<EventListener> set, Event evt, Object[] params) {
         assert evt != null;
 
         if (!F.isEmpty(set)) {
             assert set != null;
 
-            for (DiscoveryEventListener lsnr : set) {
+            for (EventListener lsnr : set) {
                 try {
-                    lsnr.onEvent(evt, cache);
+                    ((ListenerWrapper)lsnr).onEvent(evt, params);
                 }
                 catch (Throwable e) {
                     U.error(log, "Unexpected exception in listener notification for event: " + evt, e);
@@ -1307,10 +1257,93 @@ private class RequestListener implements GridMessageListener {
         }
     }
 
+    /** */
+    private abstract static class ListenerWrapper implements EventListener {
+        abstract void onEvent(Event evt, Object[] params);
+    }
+
+    /**
+     * Wraps local listener
+     */
+    private static final class LocalListenerWrapper extends ListenerWrapper {
+        /** */
+        private final GridLocalEventListener lsnr;
+
+        /**
+         * @param lsnr Listener.
+         */
+        private LocalListenerWrapper(GridLocalEventListener lsnr) {
+            this.lsnr = lsnr;
+        }
+
+        /** {@inheritDoc} */
+        @Override void onEvent(Event evt, Object[] params) {
+            lsnr.onEvent(evt);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            LocalListenerWrapper wrapper = (LocalListenerWrapper)o;
+
+            return lsnr.equals(wrapper.lsnr);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return lsnr.hashCode();
+        }
+    }
+
+    /**
+     * Wraps discovery local listener
+     */
+    private static final class DiscoveryListenerWrapper extends ListenerWrapper {
+        /** */
+        private final DiscoveryEventListener lsnr;
+
+        /**
+         * @param lsnr Listener.
+         */
+        private DiscoveryListenerWrapper(DiscoveryEventListener lsnr) {
+            this.lsnr = lsnr;
+        }
+
+        /** {@inheritDoc} */
+        @Override void onEvent(Event evt, Object[] params) {
+            // No checks there since only DiscoveryManager produses DiscoveryEvents
+            // and it uses an overloaded method with additional parameters
+            lsnr.onEvent((DiscoveryEvent)evt, (DiscoCache)params[0]);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            DiscoveryListenerWrapper wrapper = (DiscoveryListenerWrapper)o;
+
+            return lsnr.equals(wrapper.lsnr);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return lsnr.hashCode();
+        }
+    }
+
     /**
-     * Wraps user listener predicate provided via {@link org.apache.ignite.IgniteEvents#localListen(org.apache.ignite.lang.IgnitePredicate, int...)}.
+     * Wraps user listener predicate provided via {@link IgniteEvents#localListen(IgnitePredicate, int...)}.
      */
-    private class UserListenerWrapper implements GridLocalEventListener {
+    private final class UserListenerWrapper extends ListenerWrapper {
         /** */
         private final IgnitePredicate<Event> lsnr;
 
@@ -1329,9 +1362,9 @@ private IgnitePredicate<? extends Event> listener() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onEvent(Event evt) {
+        @Override void onEvent(Event evt, Object[] params) {
             if (!lsnr.apply(evt))
-                removeLocalEventListener(this);
+                removeEventListener(this, null);
         }
 
         /** {@inheritDoc} */

From 405ce563fb7c35627c6e1bb0b68f423ba089c6f2 Mon Sep 17 00:00:00 2001
From: Dmitriy Shabalin <dmitriyff@gmail.com>
Date: Fri, 14 Apr 2017 17:55:38 +0700
Subject: [PATCH 121/516] IGNITE-4068 Added common primitive for buttons group.
 Refactored existing button groups. (cherry picked from commit e5200c2)

---
 .../frontend/app/helpers/jade/mixins.pug      | 12 ++++---
 .../app/modules/sql/sql.controller.js         |  6 ----
 .../app/primitives/btn-group/index.pug        | 35 +++++++++++++++++++
 .../views/configuration/domains.tpl.pug       |  1 -
 .../frontend/views/sql/sql.tpl.pug            | 12 +++----
 5 files changed, 46 insertions(+), 20 deletions(-)
 create mode 100644 modules/web-console/frontend/app/primitives/btn-group/index.pug

diff --git a/modules/web-console/frontend/app/helpers/jade/mixins.pug b/modules/web-console/frontend/app/helpers/jade/mixins.pug
index db175a2146ef1..b8cbfe2100b7d 100644
--- a/modules/web-console/frontend/app/helpers/jade/mixins.pug
+++ b/modules/web-console/frontend/app/helpers/jade/mixins.pug
@@ -15,6 +15,7 @@
     limitations under the License.
 
 include ./form
+include ../../primitives/btn-group/index
 
 //- Mixin for advanced options toggle.
 mixin advanced-options-toggle(click, cond, showMessage, hideMessage)
@@ -49,7 +50,7 @@ mixin main-table(title, rows, focusId, click, rowTemplate, searchField)
 
 //- Mixin with save, remove, clone and undo buttons.
 mixin save-remove-clone-undo-buttons(objectName)
-    -var removeTip = '"Remove current ' + objectName + '"'
+    -var removeTip = 'Remove current ' + objectName
     -var cloneTip = '"Clone current ' + objectName + '"'
     -var undoTip = '"Undo all changes for current ' + objectName + '"'
 
@@ -60,10 +61,11 @@ mixin save-remove-clone-undo-buttons(objectName)
             a.btn.btn-primary(id='save-item' ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && saveItem()' bs-tooltip='' data-title=`{{saveBtnTipText(ui.inputForm.$dirty, '${objectName}')}}` data-placement='bottom' data-trigger='hover') Save
         .panel-tip-container(ng-show='backupItem._id')
             a.btn.btn-primary(id='clone-item' ng-click='cloneItem()' bs-tooltip=cloneTip data-placement='bottom' data-trigger='hover') Clone
-        .btn-group.panel-tip-container(ng-show='backupItem._id')
-            button.btn.btn-primary(id='remove-item' ng-click='removeItem()' bs-tooltip=removeTip data-placement='bottom' data-trigger='hover') Remove
-            button.btn.dropdown-toggle.btn-primary(id='remove-item-dropdown' data-toggle='dropdown' data-container='body' bs-dropdown='[{ text: "Remove All", click: "removeAllItems()" }]' data-placement='bottom-right')
-                span.caret
+
+        -var actions = [{ text: "Remove", click: "removeItem()" }, { text: "Remove All", click: "removeAllItems()" }]
+
+        +btn-group(actions, removeTip)(ng-show='backupItem._id')
+
         .panel-tip-container(ng-show='backupItem')
             i.btn.btn-primary.fa.fa-undo(id='undo-item' ng-disabled='!ui.inputForm.$dirty' ng-click='ui.inputForm.$dirty && resetAll()' bs-tooltip=undoTip data-placement='bottom' data-trigger='hover')
 
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index 075bd559ee1e2..b1cd3d079c89c 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -271,12 +271,6 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
             {value: 3600000, label: 'hours', short: 'h'}
         ];
 
-        $scope.exportDropdown = [
-            { text: 'Export all', click: 'exportCsvAll(paragraph)' }
-            // { 'text': 'Export all to CSV', 'click': 'exportCsvAll(paragraph)' },
-            // { 'text': 'Export all to PDF', 'click': 'exportPdfAll(paragraph)' }
-        ];
-
         $scope.metadata = [];
 
         $scope.metaFilter = '';
diff --git a/modules/web-console/frontend/app/primitives/btn-group/index.pug b/modules/web-console/frontend/app/primitives/btn-group/index.pug
new file mode 100644
index 0000000000000..c047b255bc3cd
--- /dev/null
+++ b/modules/web-console/frontend/app/primitives/btn-group/index.pug
@@ -0,0 +1,35 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+mixin btn-group(actions, tip)
+    .btn-group.panel-tip-container&attributes(attributes)
+        button.btn.btn-primary(
+            ng-click=`${actions[0].click}`
+
+            bs-tooltip=''
+            data-title=tip
+
+            data-trigger='hover'
+            data-placement='bottom'
+        ) #{actions[0].text}
+        button.btn.dropdown-toggle.btn-primary(
+            bs-dropdown=`${JSON.stringify(actions)}`
+
+            data-toggle='dropdown'
+            data-container='body'
+            data-placement='bottom-right'
+        )
+            span.caret
diff --git a/modules/web-console/frontend/views/configuration/domains.tpl.pug b/modules/web-console/frontend/views/configuration/domains.tpl.pug
index 33528c7c8dd68..a8f9b5ce19c1a 100644
--- a/modules/web-console/frontend/views/configuration/domains.tpl.pug
+++ b/modules/web-console/frontend/views/configuration/domains.tpl.pug
@@ -55,7 +55,6 @@ include /app/helpers/jade/mixins
                 .panel-tip-container(bs-tooltip='' data-title='Import domain models from database' data-placement='bottom')
                     button.btn.btn-primary(ng-click='showImportDomainModal()') Import from database
                 +save-remove-clone-undo-buttons('domain model')
-                .btn-group.panel-tip-container.pull-right(bs-tooltip='' data-title='Import domain models from demo database' data-placement='bottom')
                 hr
             .bs-affix-fix
             div(bs-collapse='' data-allow-multiple='true' ng-model='ui.activePanels')
diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug
index dcfc53137f616..6b14555a68ebf 100644
--- a/modules/web-console/frontend/views/sql/sql.tpl.pug
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -134,10 +134,8 @@ mixin table-result-heading-query
                 +result-toolbar
         .col-xs-4
             .pull-right
-                .btn-group(ng-disabled='paragraph.loading')
-                    button.btn.btn-primary(ng-click='exportCsv(paragraph)' bs-tooltip data-title='{{actionTooltip(paragraph, "export", false)}}') Export
-                    button.btn.btn-primary.dropdown-toggle(id='export-item-dropdown' data-toggle='dropdown' data-container='body' bs-dropdown='exportDropdown' data-placement='bottom-right')
-                        span.caret
+                -var actions = [{ text: "Export", click: 'exportCsv(paragraph)' }, { text: 'Export all', click: 'exportCsvAll(paragraph)' }]
+                +btn-group(actions, '{{ actionTooltip(paragraph, "export", false) }}')(ng-disabled='paragraph.loading')
 
 mixin table-result-heading-scan
     .total.row
@@ -151,10 +149,8 @@ mixin table-result-heading-scan
                 +result-toolbar
         .col-xs-4
             .pull-right
-                .btn-group(ng-disabled='paragraph.loading')
-                    button.btn.btn-primary(ng-click='exportCsv(paragraph)' bs-tooltip data-title='{{actionTooltip(paragraph, "export", false)}}') Export
-                    button.btn.btn-primary.dropdown-toggle(id='export-item-dropdown' data-toggle='dropdown' data-container='body' bs-dropdown='exportDropdown' data-placement='bottom-right')
-                        span.caret
+                -var actions = [{ text: "Export", click: 'exportCsv(paragraph)' }, { text: 'Export all', click: 'exportCsvAll(paragraph)' }]
+                +btn-group(actions, '{{ actionTooltip(paragraph, "export", false) }}')(ng-disabled='paragraph.loading')
 
 mixin table-result-body
     .grid(ui-grid='paragraph.gridOptions' ui-grid-resize-columns ui-grid-exporter)

From 60cf48dc175fa288cd74d1189f0e992c9a16dc99 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Fri, 14 Apr 2017 18:00:47 +0700
Subject: [PATCH 122/516] IGNITE-4886 Catch all errors. (cherry picked from
 commit 7e8d9e8)

---
 .../internal/visor/service/VisorServiceDescriptor.java   | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java
index 83ec77d5c22e0..26f5c10dd57b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceDescriptor.java
@@ -68,7 +68,14 @@ public VisorServiceDescriptor() {
      */
     public VisorServiceDescriptor(ServiceDescriptor srvc) {
         name = srvc.name();
-        srvcCls = VisorTaskUtils.compactClass(srvc.serviceClass());
+
+        try {
+            srvcCls = VisorTaskUtils.compactClass(srvc.serviceClass());
+        }
+        catch (Throwable e) {
+            srvcCls = e.getClass().getName() + ": " + e.getMessage();
+        }
+
         totalCnt = srvc.totalCount();
         maxPerNodeCnt = srvc.maxPerNodeCount();
         cacheName = srvc.cacheName();

From 81c3ed4c0511841f7056677db6063b4eb8d2def0 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 14 Apr 2017 18:18:23 +0700
Subject: [PATCH 123/516] IGNITE-4896 Rewored GridClientNodeBean serialization.
 (cherry picked from commit a025268)

---
 .../client/message/GridClientNodeBean.java    |   2 +-
 .../top/GridTopologyCommandHandler.java       |   4 +
 ...ConfigurationCustomSerializerSelfTest.java | 147 ++++++++++++++++++
 .../IgniteBinaryObjectsTestSuite.java         |   2 +
 4 files changed, 154 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryConfigurationCustomSerializerSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/client/message/GridClientNodeBean.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/client/message/GridClientNodeBean.java
index ca8260865d9dc..f1848dfc6d0e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/client/message/GridClientNodeBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/client/message/GridClientNodeBean.java
@@ -250,7 +250,7 @@ public void setTcpPort(int tcpPort) {
 
         U.writeUuid(out, nodeId);
 
-        out.writeObject(consistentId);
+        out.writeObject(String.valueOf(consistentId));
         out.writeObject(metrics);
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
index 297785ea760f2..536ec88815471 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
@@ -52,7 +52,9 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.IgnitePortProtocol;
 
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_BINARY_CONFIGURATION;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CACHE;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_TCP_ADDRS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_TCP_HOST_NAMES;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_TCP_PORT;
@@ -291,6 +293,8 @@ private GridClientNodeBean createNodeBean(ClusterNode node, boolean mtr, boolean
             attrs.remove(ATTR_TX_CONFIG);
             attrs.remove(ATTR_SECURITY_SUBJECT);
             attrs.remove(ATTR_SECURITY_CREDENTIALS);
+            attrs.remove(ATTR_BINARY_CONFIGURATION);
+            attrs.remove(ATTR_NODE_CONSISTENT_ID);
 
             for (Iterator<Map.Entry<String, Object>> i = attrs.entrySet().iterator(); i.hasNext();) {
                 Map.Entry<String, Object> e = i.next();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryConfigurationCustomSerializerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryConfigurationCustomSerializerSelfTest.java
new file mode 100644
index 0000000000000..1da2967612c21
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryConfigurationCustomSerializerSelfTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.UUID;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinarySerializer;
+import org.apache.ignite.binary.BinaryTypeConfiguration;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.client.GridClient;
+import org.apache.ignite.internal.client.GridClientConfiguration;
+import org.apache.ignite.internal.client.GridClientFactory;
+import org.apache.ignite.internal.client.GridClientProtocol;
+import org.apache.ignite.internal.client.balancer.GridClientRoundRobinBalancer;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.node.VisorNodePingTask;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests that node will start with custom binary serializer and thin client will connect to such node.
+ */
+public class BinaryConfigurationCustomSerializerSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        BinaryConfiguration binaryCfg = new BinaryConfiguration();
+
+        BinaryTypeConfiguration btc = new BinaryTypeConfiguration("org.MyClass");
+
+        btc.setIdMapper(BinaryContext.defaultIdMapper());
+        btc.setEnum(false);
+
+        // Set custom serializer that is unknown for Optimized marshaller.
+        btc.setSerializer(new MyBinarySerializer());
+
+        binaryCfg.setTypeConfigurations(Collections.singletonList(btc));
+
+        cfg.setBinaryConfiguration(binaryCfg);
+
+        // Set custom consistent ID that unknown for Optimized marshaller.
+        cfg.setConsistentId(new MyConsistentId("test"));
+
+        cfg.setCacheConfiguration(new CacheConfiguration("TEST"));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        startGrids(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * Test that thin client will be able to connect to node with custom binary serializer and custom consistent ID.
+     *
+     * @throws Exception If failed.
+     */
+    public void testThinClientConnected() throws Exception {
+        UUID nid = ignite(0).cluster().localNode().id();
+
+        GridClientConfiguration clnCfg = new GridClientConfiguration();
+
+        clnCfg.setProtocol(GridClientProtocol.TCP);
+        clnCfg.setServers(Collections.singleton("127.0.0.1:11211"));
+        clnCfg.setBalancer(new GridClientRoundRobinBalancer());
+
+        // Start client.
+        GridClient client = GridClientFactory.start(clnCfg);
+
+        // Execute some task.
+        client.compute().execute(VisorNodePingTask.class.getName(), new VisorTaskArgument<>(nid, nid, false));
+
+        GridClientFactory.stop(client.id(), false);
+    }
+
+    /**
+     * Custom consistent ID.
+     */
+    private static class MyConsistentId implements Serializable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** Actual ID. */
+        private String id;
+
+        /**
+         * @param id Actual ID.
+         */
+        MyConsistentId(String id) {
+            this.id = id;
+        }
+
+        /**
+         * @return Consistent ID.
+         */
+        public String getId() {
+            return id;
+        }
+    }
+
+    /**
+     * Custom BinarySerializer.
+     */
+    private static class MyBinarySerializer implements BinarySerializer {
+        /** {@inheritDoc} */
+        @Override public void writeBinary(Object obj, BinaryWriter writer) throws BinaryObjectException {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readBinary(Object obj, BinaryReader reader) throws BinaryObjectException {
+            // No-op.
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
index 3496dbf381ac8..4aa22278d1d9d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.internal.binary.BinaryBasicIdMapperSelfTest;
 import org.apache.ignite.internal.binary.BinaryBasicNameMapperSelfTest;
 import org.apache.ignite.internal.binary.BinaryConfigurationConsistencySelfTest;
+import org.apache.ignite.internal.binary.BinaryConfigurationCustomSerializerSelfTest;
 import org.apache.ignite.internal.binary.BinaryEnumsSelfTest;
 import org.apache.ignite.internal.binary.BinaryFieldIdentityResolverSelfTest;
 import org.apache.ignite.internal.binary.BinaryFieldsHeapSelfTest;
@@ -100,6 +101,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(BinaryIdentityResolverConfigurationSelfTest.class);
 
         suite.addTestSuite(BinaryConfigurationConsistencySelfTest.class);
+        suite.addTestSuite(BinaryConfigurationCustomSerializerSelfTest.class);
         suite.addTestSuite(GridBinaryMarshallerCtxDisabledSelfTest.class);
         suite.addTestSuite(BinaryObjectBuilderDefaultMappersSelfTest.class);
         suite.addTestSuite(BinaryObjectBuilderSimpleNameLowerCaseMappersSelfTest.class);

From 4a1415ad01ff9fde30d5c7c02e6d938f1515178d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 12 Apr 2017 13:01:25 +0300
Subject: [PATCH 124/516] IGNITE-4907: Fixed excessive service instances can be
 started with dynamic deployment. This closes #1766.

(cherry picked from commit 0f7ef74)
---
 .../service/GridServiceProcessor.java         |  2 +-
 ...rviceProcessorMultiNodeConfigSelfTest.java | 95 ++++++++++++++++---
 ...GridServiceProcessorMultiNodeSelfTest.java | 61 ++++++++++++
 3 files changed, 146 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index a8af9832e9355..2a363e21e8ef5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -972,7 +972,7 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
                         int perNodeCnt = totalCnt != 0 ? totalCnt / size : maxPerNodeCnt;
                         int remainder = totalCnt != 0 ? totalCnt % size : 0;
 
-                        if (perNodeCnt > maxPerNodeCnt && maxPerNodeCnt != 0) {
+                        if (perNodeCnt >= maxPerNodeCnt && maxPerNodeCnt != 0) {
                             perNodeCnt = maxPerNodeCnt;
                             remainder = 0;
                         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index 1bd3b035a446c..9da62c0abb702 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -17,11 +17,13 @@
 
 package org.apache.ignite.internal.processors.service;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
-import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -38,6 +40,9 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
     /** Node singleton name. */
     private static final String NODE_SINGLE_BUT_CLIENT = "serviceConfigEachNodeButClient";
 
+    /** Node singleton name. */
+    private static final String NODE_SINGLE_WITH_LIMIT = "serviceConfigWithLimit";
+
     /** Affinity service name. */
     private static final String AFFINITY = "serviceConfigAffinity";
 
@@ -51,7 +56,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        ServiceConfiguration[] arr = new ServiceConfiguration[4];
+        List<ServiceConfiguration> cfgs = new ArrayList<>();
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -60,7 +65,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new DummyService());
 
-        arr[0] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -68,7 +73,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setMaxPerNodeCount(1);
         cfg.setService(new DummyService());
 
-        arr[1] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -79,7 +84,7 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new AffinityService(AFFINITY_KEY));
 
-        arr[2] = cfg;
+        cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
 
@@ -88,9 +93,18 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setNodeFilter(new CacheConfiguration.IgniteAllNodesPredicate());
         cfg.setService(new DummyService());
 
-        arr[3] = cfg;
+        cfgs.add(cfg);
+
+        cfg = new ServiceConfiguration();
 
-        return arr;
+        cfg.setName(NODE_SINGLE_WITH_LIMIT);
+        cfg.setMaxPerNodeCount(1);
+        cfg.setTotalCount(nodeCount() + 1);
+        cfg.setService(new DummyService());
+
+        cfgs.add(cfg);
+
+        return cfgs.toArray(new ServiceConfiguration[cfgs.size()]);
     }
 
     /** {@inheritDoc} */
@@ -107,6 +121,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
                         DummyService.cancelled(NODE_SINGLE) == 0 &&
                         DummyService.started(NODE_SINGLE_BUT_CLIENT) == nodeCount() &&
                         DummyService.cancelled(NODE_SINGLE_BUT_CLIENT) == 0 &&
+                        DummyService.started(NODE_SINGLE_WITH_LIMIT) >= nodeCount() &&
+                        DummyService.cancelled(NODE_SINGLE_WITH_LIMIT) == 0 &&
                         actualCount(AFFINITY, randomGrid().services().serviceDescriptors()) == 1;
                 }
             },
@@ -170,6 +186,59 @@ public void testAffinityUpdateTopology() throws Exception {
         finally {
             stopExtraNodes(nodeCnt);
         }
+
+        checkCount(AFFINITY, g.services().serviceDescriptors(), 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployLimits() throws Exception {
+        final Ignite g = randomGrid();
+
+        final String name = NODE_SINGLE_WITH_LIMIT;
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
+
+        int extraNodes = 2;
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        DummyService.exeLatch(name, latch);
+
+        startExtraNodes(extraNodes);
+
+        try {
+            latch.await();
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount() + 1);
+        }
+        finally {
+            stopExtraNodes(extraNodes);
+        }
+
+        assertEquals(name, 1, DummyService.cancelled(name));
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
+    }
+
+    /**
+     * @param srvcName Service name
+     * @param expectedDeps Expected number of service deployments
+     *
+     */
+    private boolean waitForDeployment(final String srvcName, final int expectedDeps) throws IgniteInterruptedCheckedException {
+        final Ignite g = randomGrid();
+
+        return GridTestUtils.waitForCondition(new GridAbsPredicateX() {
+            @Override public boolean applyx() {
+                return actualCount(srvcName, g.services().serviceDescriptors())  == expectedDeps;
+            }
+        }, 1500);
     }
 
     /**
@@ -212,10 +281,6 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         try {
             latch.await();
 
-            // Ensure service is deployed.
-            assertNotNull(grid(nodeCount() + newNodes - 1).services()
-                .serviceProxy(NODE_SINGLE_BUT_CLIENT, Service.class, false, 2000));
-
             assertEquals(name, newNodes, DummyService.started(name));
             assertEquals(name, 0, DummyService.cancelled(name));
 
@@ -224,6 +289,10 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         finally {
             stopExtraNodes(newNodes);
         }
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 
     /**
@@ -253,5 +322,9 @@ private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Ex
         finally {
             stopExtraNodes(servers + clients);
         }
+
+        waitForDeployment(name, nodeCount());
+
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index f7403dcc820ca..d133cf299a756 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -264,4 +264,65 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
             stopGrid("client");
         }
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployLimits() throws Exception {
+            String name = "serviceWithLimitsUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            final int totalInstances = nodeCount() + 1;
+
+            CountDownLatch latch = new CountDownLatch(nodeCount());
+
+            DummyService.exeLatch(name, latch);
+
+            ServiceConfiguration srvcCfg = new ServiceConfiguration();
+
+            srvcCfg.setName(name);
+            srvcCfg.setMaxPerNodeCount(1);
+            srvcCfg.setTotalCount(totalInstances);
+            srvcCfg.setService(new DummyService());
+
+            IgniteServices svcs = g.services().withAsync();
+
+            svcs.deploy(srvcCfg);
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
+            latch.await();
+
+            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
+            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+            checkCount(name, g.services().serviceDescriptors(), nodeCount());
+
+            int extraNodes = 2;
+
+            latch = new CountDownLatch(1);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(2);
+
+            try {
+                latch.await();
+
+                TestCase.assertEquals(name, totalInstances, DummyService.started(name));
+                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), totalInstances);
+            }
+            finally {
+                stopExtraNodes(extraNodes);
+            }
+    }
 }
\ No newline at end of file

From e206b9f1fd3dbf927f940d558144a4796479ed5d Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgain.com>
Date: Fri, 14 Apr 2017 18:32:30 +0700
Subject: [PATCH 125/516] IGNITE-4871 Added Kubernetes IP finder to Cluster
 configuration screen. (cherry picked from commit f978ff2)

---
 modules/web-console/backend/app/mongo.js      |  8 +++-
 .../frontend/app/data/pom-dependencies.json   |  1 +
 .../app/helpers/jade/form/form-field-text.pug | 40 +++++++++++--------
 .../frontend/app/helpers/jade/mixins.pug      |  8 +++-
 .../generator/ConfigurationGenerator.js       | 12 ++++++
 .../generator/defaults/Cluster.service.js     |  6 +++
 .../states/configuration/clusters/general.pug |  3 ++
 .../clusters/general/discovery/kubernetes.pug | 37 +++++++++++++++++
 .../summary/summary.controller.js             |  3 ++
 .../configuration/summary/summary.worker.js   | 22 ++++++++++
 .../controllers/clusters-controller.js        |  3 +-
 11 files changed, 124 insertions(+), 19 deletions(-)
 create mode 100644 modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/kubernetes.pug

diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index 2d252b9b8eccc..f4a62b1fe2558 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -407,7 +407,7 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
             authenticator: String,
             forceServerMode: Boolean,
             clientReconnectDisabled: Boolean,
-            kind: {type: String, enum: ['Vm', 'Multicast', 'S3', 'Cloud', 'GoogleStorage', 'Jdbc', 'SharedFs', 'ZooKeeper']},
+            kind: {type: String, enum: ['Vm', 'Multicast', 'S3', 'Cloud', 'GoogleStorage', 'Jdbc', 'SharedFs', 'ZooKeeper', 'Kubernetes']},
             Vm: {
                 addresses: [String]
             },
@@ -528,6 +528,12 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
                 basePath: String,
                 serviceName: String,
                 allowDuplicateRegistrations: Boolean
+            },
+            Kubernetes: {
+                serviceName: String,
+                namespace: String,
+                masterUrl: String,
+                accountToken: String
             }
         },
         atomicConfiguration: {
diff --git a/modules/web-console/frontend/app/data/pom-dependencies.json b/modules/web-console/frontend/app/data/pom-dependencies.json
index 7d2bed0de8172..3e0543bb64861 100644
--- a/modules/web-console/frontend/app/data/pom-dependencies.json
+++ b/modules/web-console/frontend/app/data/pom-dependencies.json
@@ -3,6 +3,7 @@
     "S3": {"artifactId": "ignite-aws"},
     "GoogleStorage": {"artifactId": "ignite-gce"},
     "ZooKeeper": {"artifactId": "ignite-zookeeper"},
+    "Kubernetes": {"artifactId": "ignite-kubernetes"},
 
     "Log4j": {"artifactId": "ignite-log4j"},
     "Log4j2": {"artifactId": "ignite-log4j2"},
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
index 76ea6e609d6e1..63feaf85f13a3 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
@@ -14,12 +14,12 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin ignite-form-field-input(name, model, disabled, required, placeholder)
+mixin ignite-field-input(type, name, model, disabled, required, placeholder)
     input.form-control(
         id=`{{ ${name} }}Input`
         name=`{{ ${name} }}`
         placeholder=placeholder
-        type='text'
+        type=type
 
         data-ng-model=model
 
@@ -30,21 +30,11 @@ mixin ignite-form-field-input(name, model, disabled, required, placeholder)
         data-ignite-form-panel-field=''
     )&attributes(attributes ? attributes.attributes ? attributes.attributes : attributes: {})
 
-mixin ignite-form-field-url-input(name, model, disabled, required, placeholder)
-    input.form-control(
-        id=`{{ ${name} }}Input`
-        name=`{{ ${name} }}`
-        placeholder=placeholder
-        type='url'
-
-        data-ng-model=model
-
-        data-ng-required=required && `${required}`
-        data-ng-disabled=disabled && `${disabled}`
-        data-ng-focus='tableReset()'
+mixin ignite-form-field-input(name, model, disabled, required, placeholder)
+    +ignite-field-input('text', name, model, disabled, required, placeholder)
 
-        data-ignite-form-panel-field=''
-    )&attributes(attributes ? attributes.attributes ? attributes.attributes : attributes: {})
+mixin ignite-form-field-url-input(name, model, disabled, required, placeholder)
+    +ignite-field-input('url', name, model, disabled, required, placeholder)
 
 mixin ignite-form-field-text(label, model, name, disabled, required, placeholder, tip)
     -var errLbl = label.substring(0, label.length - 1)
@@ -62,3 +52,21 @@ mixin ignite-form-field-text(label, model, name, disabled, required, placeholder
 
             .input-tip
                 +ignite-form-field-input(name, model, disabled, required, placeholder)(attributes=attributes)
+
+mixin ignite-form-field-url(label, model, name, required, placeholder, tip)
+    -var errLbl = label.substring(0, label.length - 1)
+
+    .ignite-form-field
+        +ignite-form-field__label(label, name, required)
+        .ignite-form-field__control
+            if tip
+                i.tipField.icon-help(bs-tooltip='' data-title=tip)
+
+            if block
+                block
+
+            +form-field-feedback(name, 'required', errLbl + ' could not be empty!')
+            +form-field-feedback(name, 'url', errLbl + ' should be a valid URL!')
+
+            .input-tip
+                +ignite-form-field-url-input(name, model, false, required, placeholder)(attributes=attributes)
diff --git a/modules/web-console/frontend/app/helpers/jade/mixins.pug b/modules/web-console/frontend/app/helpers/jade/mixins.pug
index b8cbfe2100b7d..472d30414b31e 100644
--- a/modules/web-console/frontend/app/helpers/jade/mixins.pug
+++ b/modules/web-console/frontend/app/helpers/jade/mixins.pug
@@ -199,6 +199,12 @@ mixin text(lbl, model, name, required, placeholder, tip)
         if  block
             block
 
+//- Mixin for text field.
+mixin url(lbl, model, name, required, placeholder, tip)
+    +ignite-form-field-url(lbl, model, name, required, placeholder, tip)
+        if  block
+            block
+
 //- Mixin for password field.
 mixin password(lbl, model, name, required, placeholder, tip)
     +ignite-form-field-password(lbl, model, name, false, required, placeholder, tip)
@@ -350,7 +356,7 @@ mixin table-java-package-field(name, model, items, valid, save, newItem)
                 ignite-on-escape=onEscape
             )
 
-//- Mixin for table java package field.
+//- Mixin for table url field.
 mixin table-url-field(name, model, items, valid, save, newItem)
     -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
     -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 670b8284f14c1..7cd910da43f93 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -320,6 +320,18 @@ export default class IgniteConfigurationGenerator {
                     .boolProperty('allowDuplicateRegistrations');
 
                 break;
+
+            case 'Kubernetes':
+                ipFinder = new Bean('org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder',
+                    'ipFinder', cluster.discovery.Kubernetes, clusterDflts.discovery.Kubernetes);
+
+                ipFinder.stringProperty('serviceName')
+                    .stringProperty('namespace')
+                    .stringProperty('masterUrl')
+                    .pathProperty('accountToken');
+
+                break;
+
             default:
                 // No-op.
         }
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
index 6333ef96afcd1..b2e91c8836fe8 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
@@ -75,6 +75,12 @@ const DFLT_CLUSTER = {
             Forever: {
                 retryIntervalMs: 1000
             }
+        },
+        Kubernetes: {
+            serviceName: 'ignite',
+            namespace: 'default',
+            masterUrl: 'https://kubernetes.default.svc.cluster.local:443',
+            accountToken: '/var/run/secrets/kubernetes.io/serviceaccount/token'
         }
     },
     atomics: {
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
index dfc49d61ef4c2..62cc23db0045a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
@@ -52,6 +52,7 @@ include /app/helpers/jade/mixins
                             <li>JDBC - JDBC based IP finder that use database to store node IP address</li>\
                             <li>Shared filesystem - Shared filesystem based IP finder that use file to store node IP address</li>\
                             <li>Apache ZooKeeper - Apache ZooKeeper based IP finder when you use ZooKeeper to coordinate your distributed environment</li>\
+                            <li>Kubernetes - IP finder for automatic lookup of Ignite nodes running in Kubernetes environment</li>\
                         </ul>')
                 .settings-row
                     .panel-details
@@ -71,6 +72,8 @@ include /app/helpers/jade/mixins
                             include ./general/discovery/vm
                         div(ng-if=`${modelDiscoveryKind} === 'ZooKeeper'`)
                             include ./general/discovery/zookeeper
+                        div(ng-if=`${modelDiscoveryKind} === 'Kubernetes'`)
+                            include ./general/discovery/kubernetes
             .col-sm-6
                 -var model = 'backupItem'
                 +preview-xml-java(model, 'clusterCaches', 'caches')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/kubernetes.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/kubernetes.pug
new file mode 100644
index 0000000000000..c4e639816cd58
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/kubernetes.pug
@@ -0,0 +1,37 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins
+
+-var discoveryKind = 'Kubernetes'
+-var model = 'backupItem.discovery.Kubernetes'
+
+div
+    .details-row
+        +text('Service name:', `${model}.serviceName`, `'${discoveryKind}ServiceName'`, 'false', 'ignite',
+            "The name of Kubernetes service for Ignite pods' IP addresses lookup.<br/>\
+            The name of the service must be equal to the name set in service's Kubernetes configuration.<br/>\
+            If this parameter is not changed then the name of the service has to be set to 'ignite' in the corresponding Kubernetes configuration.")
+    .details-row
+        +text('Namespace:', `${model}.namespace`, `'${discoveryKind}Namespace'`, 'false', 'default',
+            "The namespace the Kubernetes service belongs to.<br/>\
+            By default, it's supposed that the service is running under Kubernetes `default` namespace.")
+    .details-row
+        +url('Kubernetes server:', `${model}.masterUrl`, `'${discoveryKind}MasterUrl'`, 'false', 'https://kubernetes.default.svc.cluster.local:443',
+            'The host name of the Kubernetes API server')
+    .details-row
+        +text('Service token file:', `${model}.accountToken`, `'${discoveryKind}AccountToken'`, 'false', '/var/run/secrets/kubernetes.io/serviceaccount/token',
+            'The path to the service token file')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
index 25203b30d1a27..0089528140f53 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.controller.js
@@ -264,6 +264,9 @@ export default [
             if (cluster.discovery.kind === 'Jdbc' && cluster.discovery.Jdbc.dialect)
                 $scope.dialects[cluster.discovery.Jdbc.dialect] = true;
 
+            if (cluster.discovery.kind === 'Kubernetes')
+                resourcesFolder.children.push({ type: 'file', name: 'ignite-service.yaml' });
+
             _.forEach(cluster.caches, (cache) => {
                 if (cache.cacheStoreFactory) {
                     const store = cache.cacheStoreFactory[cache.cacheStoreFactory.kind];
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
index 7ea1d5ad5beb0..5933f6caa6c53 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
@@ -38,6 +38,25 @@ const generator = IgniteConfigurationGenerator;
 
 const escapeFileName = (name) => name.replace(/[\\\/*\"\[\],\.:;|=<>?]/g, '-').replace(/ /g, '_');
 
+const kubernetesConfig = (cluster) => {
+    if (!cluster.discovery.Kubernetes)
+        cluster.discovery.Kubernetes = { serviceName: 'ignite' };
+
+    return `apiVersion: v1\n\
+kind: Service\n\
+metadata:\n\
+  # Name of Ignite Service used by Kubernetes IP finder for IP addresses lookup.\n\
+  name: ${ cluster.discovery.Kubernetes.serviceName || 'ignite' }\n\
+spec:\n\
+  clusterIP: None # custom value.\n\
+  ports:\n\
+    - port: 9042 # custom value.\n\
+  selector:\n\
+    # Must be equal to one of the labels set in Ignite pods'\n\
+    # deployement configuration.\n\
+    app: ${ cluster.discovery.Kubernetes.serviceName || 'ignite' }`;
+};
+
 // eslint-disable-next-line no-undef
 onmessage = function(e) {
     const {cluster, data, demo} = e.data;
@@ -67,6 +86,9 @@ onmessage = function(e) {
 
     const metaPath = `${resourcesPath}/META-INF`;
 
+    if (cluster.discovery.kind === 'Kubernetes')
+        zip.file(`${metaPath}/ignite-service.yaml`, kubernetesConfig(cluster));
+
     zip.file(`${metaPath}/${serverXml}`, spring.igniteConfiguration(cfg).asString());
     zip.file(`${metaPath}/${clientXml}`, spring.igniteConfiguration(clientCfg, clientNearCaches).asString());
 
diff --git a/modules/web-console/frontend/controllers/clusters-controller.js b/modules/web-console/frontend/controllers/clusters-controller.js
index c8392cf7ef0aa..e936955e05054 100644
--- a/modules/web-console/frontend/controllers/clusters-controller.js
+++ b/modules/web-console/frontend/controllers/clusters-controller.js
@@ -228,7 +228,8 @@ export default ['clustersController', [
             {value: 'GoogleStorage', label: 'Google cloud storage'},
             {value: 'Jdbc', label: 'JDBC'},
             {value: 'SharedFs', label: 'Shared filesystem'},
-            {value: 'ZooKeeper', label: 'Apache ZooKeeper'}
+            {value: 'ZooKeeper', label: 'Apache ZooKeeper'},
+            {value: 'Kubernetes', label: 'Kubernetes'}
         ];
 
         $scope.swapSpaceSpis = [

From 4518f51f4c1c29517d518f5e44f5d70d17c51170 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 14 Apr 2017 16:38:46 +0300
Subject: [PATCH 126/516] Added IgniteGetFromComputeBenchmark.

---
 .../cache/IgniteGetFromComputeBenchmark.java  | 167 ++++++++++++++++++
 1 file changed, 167 insertions(+)
 create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetFromComputeBenchmark.java

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetFromComputeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetFromComputeBenchmark.java
new file mode 100644
index 0000000000000..66aec82653d2f
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetFromComputeBenchmark.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.yardstick.cache.model.SampleValue;
+import org.yardstickframework.BenchmarkConfiguration;
+
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Benchmark created to verify that slow EntryProcessor does not affect 'get' performance.
+ */
+public class IgniteGetFromComputeBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
+    /** */
+    private static final String CACHE_NAME = "atomic-offheap";
+
+    /** */
+    private IgniteCompute compute;
+
+    /** */
+    private IgniteCache asyncCache;
+
+    /** */
+    private ThreadLocal<IgniteFuture> invokeFut = new ThreadLocal<>();
+
+    /** {@inheritDoc} */
+    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        if (args.preloadAmount() > args.range())
+            throw new IllegalArgumentException("Preloading amount (\"-pa\", \"--preloadAmount\") " +
+                "must by less then the range (\"-r\", \"--range\").");
+
+        String cacheName = cache().getName();
+
+        println(cfg, "Loading data for cache: " + cacheName);
+
+        long start = System.nanoTime();
+
+        try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(cacheName)) {
+            for (int i = 0; i < args.preloadAmount(); i++) {
+                dataLdr.addData(i, new SampleValue(i));
+
+                if (i % 100000 == 0) {
+                    if (Thread.currentThread().isInterrupted())
+                        break;
+
+                    println("Loaded entries: " + i);
+                }
+            }
+        }
+
+        println(cfg, "Finished populating data [time=" + ((System.nanoTime() - start) / 1_000_000) + "ms, " +
+            "amount=" + args.preloadAmount() + ']');
+
+        compute = ignite().compute();
+
+        asyncCache = cache().withAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        IgniteFuture fut = invokeFut.get();
+
+        if (fut == null || fut.isDone()) {
+            Set<Integer> keys = new TreeSet<>();
+
+            for (int i = 0; i < 3; i++)
+                keys.add(nextRandom(args.range()));
+
+            asyncCache.invokeAll(keys, new SlowEntryProcessor(0));
+
+            invokeFut.set(asyncCache.future());
+        }
+
+        int key = nextRandom(args.range());
+
+        compute.affinityCall(CACHE_NAME, key, new GetClosure(key));
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteCache<Integer, Object> cache() {
+        return ignite().cache(CACHE_NAME);
+    }
+
+    /**
+     *
+     */
+    public static class GetClosure implements IgniteCallable<Object> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** */
+        private final int key;
+
+        /**
+         * @param key Key.
+         */
+        public GetClosure(int key) {
+            this.key = key;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object call() throws Exception {
+            return ignite.cache(CACHE_NAME).get(key);
+        }
+    }
+
+    /**
+     *
+     */
+    public static class SlowEntryProcessor implements CacheEntryProcessor<Integer, Object, Object> {
+        /** */
+        private Object val;
+
+        /**
+         * @param val Value.
+         */
+        public SlowEntryProcessor(Object val) {
+            this.val = val;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object process(MutableEntry<Integer, Object> entry, Object... args) {
+            try {
+                Thread.sleep(10);
+            }
+            catch (InterruptedException ignore) {
+                // No-op.
+            }
+
+            entry.setValue(val);
+
+            return null;
+        }
+    }
+}

From b22738080101536a8af1ed60e70d693897e9bc7c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 14 Apr 2017 17:54:02 +0300
Subject: [PATCH 127/516] ignite-4173 Fix test. Permits must be released on
 node fail.

(cherry picked from commit 1f867c6)
---
 .../GridCacheAbstractDataStructuresFailoverSelfTest.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
index 9295770796381..4ab7b6780d2aa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractDataStructuresFailoverSelfTest.java
@@ -418,7 +418,7 @@ public void testSemaphoreFailoverSafe() throws Exception {
 
             stopGrid(NEW_GRID_NAME);
 
-            assertEquals(10, semaphore.availablePermits());
+            assertEquals(20, semaphore.availablePermits());
         }
     }
 

From 1985496ea98f4d7112a0b99727297427f343e9b2 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 17 Apr 2017 10:20:41 +0300
Subject: [PATCH 128/516] Add missed license headers.

---
 modules/kubernetes/config/Dockerfile            | 17 +++++++++++++++++
 .../kubernetes/config/ignite-deployment.yaml    | 17 +++++++++++++++++
 modules/kubernetes/config/ignite-service.yaml   | 17 +++++++++++++++++
 3 files changed, 51 insertions(+)

diff --git a/modules/kubernetes/config/Dockerfile b/modules/kubernetes/config/Dockerfile
index 4e08ce80e2147..2274535b5ccb2 100644
--- a/modules/kubernetes/config/Dockerfile
+++ b/modules/kubernetes/config/Dockerfile
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # Use Java 8 image as default one.
 FROM java:8
 
diff --git a/modules/kubernetes/config/ignite-deployment.yaml b/modules/kubernetes/config/ignite-deployment.yaml
index ed5c102a666a6..1e30f075e63bb 100644
--- a/modules/kubernetes/config/ignite-deployment.yaml
+++ b/modules/kubernetes/config/ignite-deployment.yaml
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # An example of a Kubernetes configuration for Ignite pods deployment.
 apiVersion: extensions/v1beta1
 kind: Deployment
diff --git a/modules/kubernetes/config/ignite-service.yaml b/modules/kubernetes/config/ignite-service.yaml
index 07b751624cbbf..7a7343ab522b3 100644
--- a/modules/kubernetes/config/ignite-service.yaml
+++ b/modules/kubernetes/config/ignite-service.yaml
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # An example of a Kubernetes configuration for Ignite lookup service deployment.
 apiVersion: v1
 kind: Service

From ae794ab45b22abe1e570d5d29661396ea7c0f951 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 17 Apr 2017 13:40:28 +0300
Subject: [PATCH 129/516] IGNITE-4159 Fix version.

---
 modules/kubernetes/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/kubernetes/pom.xml b/modules/kubernetes/pom.xml
index 5d4e5f0c8b509..7a632c617934f 100644
--- a/modules/kubernetes/pom.xml
+++ b/modules/kubernetes/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kubernetes</artifactId>
-    <version>2.0.0-SNAPSHOT</version>
+    <version>1.9.0-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>

From bf1049741f7a64728bd433f78262ba273f969848 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 17 Apr 2017 19:00:30 +0300
Subject: [PATCH 130/516] IGNITE-4954 - Configurable expiration timeout for
 Cassandra session. This closes #1785.

---
 .../cassandra/datasource/DataSource.java      | 26 ++++++++++++++++++-
 .../session/CassandraSessionImpl.java         | 23 +++++++++-------
 .../cassandra/session/pool/SessionPool.java   |  6 ++---
 .../session/pool/SessionWrapper.java          | 15 ++++++-----
 4 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
index 915eebde73f01..1fa2a1df9faa4 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
@@ -46,6 +46,9 @@
  * Data source abstraction to specify configuration of the Cassandra session to be used.
  */
 public class DataSource {
+    /** Default expiration timeout for Cassandra driver session. */
+    public static final long DFLT_SESSION_EXPIRATION_TIMEOUT = 300000; // 5 minutes.
+
     /** Number of rows to immediately fetch in CQL statement execution. */
     private Integer fetchSize;
 
@@ -123,6 +126,9 @@ public class DataSource {
     /** Netty options to use for connection. */
     private NettyOptions nettyOptions;
 
+    /** Expiration timeout for Cassandra driver session. */
+    private long sessionExpirationTimeout = DFLT_SESSION_EXPIRATION_TIMEOUT;
+
     /** Cassandra session wrapper instance. */
     private volatile CassandraSession ses;
 
@@ -441,6 +447,23 @@ public void setNettyOptions(NettyOptions options) {
         invalidate();
     }
 
+    /**
+     * Sets expiration timeout for Cassandra driver session. Idle sessions that are not
+     * used during this timeout value will be automatically closed and recreated later
+     * on demand.
+     * <p>
+     * If set to {@code 0}, timeout is disabled.
+     * <p>
+     * Default value is {@link #DFLT_SESSION_EXPIRATION_TIMEOUT}.
+     *
+     * @param sessionExpirationTimeout Expiration timeout for Cassandra driver session.
+     */
+    public void setSessionExpirationTimeout(long sessionExpirationTimeout) {
+        this.sessionExpirationTimeout = sessionExpirationTimeout;
+
+        invalidate();
+    }
+
     /**
      * Creates Cassandra session wrapper if it wasn't created yet and returns it
      *
@@ -523,7 +546,8 @@ public synchronized CassandraSession session(IgniteLogger log) {
         if (nettyOptions != null)
             builder = builder.withNettyOptions(nettyOptions);
 
-        return ses = new CassandraSessionImpl(builder, fetchSize, readConsistency, writeConsistency, log);
+        return ses = new CassandraSessionImpl(
+            builder, fetchSize, readConsistency, writeConsistency, sessionExpirationTimeout, log);
     }
 
     /**
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java
index 95b8581c5c793..cee776bb27363 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java
@@ -17,6 +17,13 @@
 
 package org.apache.ignite.cache.store.cassandra.session;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.Cache;
 import com.datastax.driver.core.BatchStatement;
 import com.datastax.driver.core.BoundStatement;
 import com.datastax.driver.core.Cluster;
@@ -30,13 +37,6 @@
 import com.datastax.driver.core.exceptions.AlreadyExistsException;
 import com.datastax.driver.core.exceptions.InvalidQueryException;
 import com.datastax.driver.core.querybuilder.Batch;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.cache.Cache;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.store.cassandra.common.CassandraHelper;
@@ -82,6 +82,9 @@ public class CassandraSessionImpl implements CassandraSession {
     /** Consistency level for Cassandra WRITE operations (insert/update/delete). */
     private ConsistencyLevel writeConsistency;
 
+    /** Expiration timeout. */
+    private long expirationTimeout;
+
     /** Logger. */
     private IgniteLogger log;
 
@@ -101,11 +104,12 @@ public class CassandraSessionImpl implements CassandraSession {
      * @param log Logger.
      */
     public CassandraSessionImpl(Cluster.Builder builder, Integer fetchSize, ConsistencyLevel readConsistency,
-        ConsistencyLevel writeConsistency, IgniteLogger log) {
+        ConsistencyLevel writeConsistency, long expirationTimeout, IgniteLogger log) {
         this.builder = builder;
         this.fetchSize = fetchSize;
         this.readConsistency = readConsistency;
         this.writeConsistency = writeConsistency;
+        this.expirationTimeout = expirationTimeout;
         this.log = log;
     }
 
@@ -404,7 +408,8 @@ else if (CassandraHelper.isPreparedStatementClusterError(e))
     /** {@inheritDoc} */
     @Override public synchronized void close() throws IOException {
         if (decrementSessionRefs() == 0 && ses != null) {
-            SessionPool.put(this, ses);
+            SessionPool.put(this, ses, expirationTimeout);
+
             ses = null;
         }
     }
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java
index fc4a907e6e06e..86db713482160 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java
@@ -17,13 +17,13 @@
 
 package org.apache.ignite.cache.store.cassandra.session.pool;
 
-import com.datastax.driver.core.Session;
 import java.lang.Thread.State;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import com.datastax.driver.core.Session;
 import org.apache.ignite.cache.store.cassandra.session.CassandraSessionImpl;
 
 /**
@@ -98,14 +98,14 @@ private static class SessionMonitor extends Thread {
      * @param cassandraSes Session wrapper.
      * @param driverSes Driver session.
      */
-    public static void put(CassandraSessionImpl cassandraSes, Session driverSes) {
+    public static void put(CassandraSessionImpl cassandraSes, Session driverSes, long expirationTimeout) {
         if (cassandraSes == null || driverSes == null)
             return;
 
         SessionWrapper old;
 
         synchronized (sessions) {
-            old = sessions.put(cassandraSes, new SessionWrapper(driverSes));
+            old = sessions.put(cassandraSes, new SessionWrapper(driverSes, expirationTimeout));
 
             if (monitorSingleton == null || State.TERMINATED.equals(monitorSingleton.getState())) {
                 monitorSingleton = new SessionMonitor();
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java
index 7c5722bb8f21f..68b9dd480f8b8 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java
@@ -24,12 +24,12 @@
  * Wrapper for Cassandra driver session, responsible for monitoring session expiration and its closing.
  */
 public class SessionWrapper {
-    /** Expiration timeout for Cassandra driver session. */
-    public static final long DFLT_EXPIRATION_TIMEOUT = 300000;  // 5 minutes.
-
     /** Cassandra driver session. */
     private Session ses;
 
+    /** Expiration timeout. */
+    private long expirationTimeout;
+
     /** Wrapper creation time.  */
     private long time;
 
@@ -38,9 +38,11 @@ public class SessionWrapper {
      *
      * @param ses Cassandra driver session.
      */
-    public SessionWrapper(Session ses) {
+    public SessionWrapper(Session ses, long expirationTimeout) {
         this.ses = ses;
-        this.time = System.currentTimeMillis();
+        this.expirationTimeout = expirationTimeout;
+
+        time = System.currentTimeMillis();
     }
 
     /**
@@ -49,7 +51,7 @@ public SessionWrapper(Session ses) {
      * @return true if session expired.
      */
     public boolean expired() {
-        return System.currentTimeMillis() - time > DFLT_EXPIRATION_TIMEOUT;
+        return expirationTimeout > 0 && System.currentTimeMillis() - time > expirationTimeout;
     }
 
     /**
@@ -66,6 +68,7 @@ public Session driverSession() {
      */
     public void release() {
         CassandraHelper.closeSession(ses);
+
         ses = null;
     }
 }

From 86c49514c64225059a31912fdccadc6dc68a8397 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 18 Apr 2017 11:15:44 +0300
Subject: [PATCH 131/516] ignite-4932 For offheap_tiered cache first try read
 value without cache entry creation

---
 .../JettyRestProcessorAbstractSelfTest.java   |  19 +
 .../processors/cache/GridCacheAdapter.java    | 142 +++++--
 .../cache/GridCacheConcurrentMap.java         |   2 -
 .../cache/GridCacheConcurrentMapImpl.java     |   7 +-
 .../processors/cache/GridCacheContext.java    |  15 +
 .../cache/GridCacheEventManager.java          |  32 ++
 .../processors/cache/GridCacheMapEntry.java   |  12 +-
 .../cache/GridCacheMapEntryFactory.java       |   4 +-
 .../cache/GridCacheSwapManager.java           |  36 ++
 .../cache/GridNoStorageCacheMap.java          |   6 +-
 .../cache/IgniteCacheExpiryPolicy.java        |   5 +
 .../GridDistributedCacheEntry.java            |   6 +-
 .../GridCachePartitionedConcurrentMap.java    |   4 +-
 .../distributed/dht/GridDhtCacheAdapter.java  |   9 +-
 .../distributed/dht/GridDhtCacheEntry.java    |   6 +-
 .../dht/GridDhtLocalPartition.java            |   4 +-
 .../dht/GridDhtOffHeapCacheEntry.java         |   6 +-
 .../dht/GridPartitionedGetFuture.java         | 166 +++++---
 .../dht/GridPartitionedSingleGetFuture.java   | 153 ++++---
 .../dht/atomic/GridDhtAtomicCache.java        | 237 ++++++-----
 .../dht/atomic/GridDhtAtomicCacheEntry.java   |   6 +-
 .../GridDhtAtomicOffHeapCacheEntry.java       |   6 +-
 .../dht/colocated/GridDhtColocatedCache.java  | 232 ++++++-----
 .../colocated/GridDhtColocatedCacheEntry.java |   6 +-
 .../GridDhtColocatedOffHeapCacheEntry.java    |   6 +-
 .../colocated/GridDhtDetachedCacheEntry.java  |   8 +-
 .../near/GridNearCacheAdapter.java            |   7 +-
 .../distributed/near/GridNearCacheEntry.java  |   6 +-
 .../near/GridNearOffHeapCacheEntry.java       |   6 +-
 .../cache/local/GridLocalCache.java           |   5 +-
 .../cache/local/GridLocalCacheEntry.java      |   6 +-
 .../local/atomic/GridLocalAtomicCache.java    | 195 +++++----
 .../transactions/IgniteTxLocalAdapter.java    |   8 +-
 .../cache/CacheRebalancingSelfTest.java       |  17 +-
 .../cache/IgniteCacheNoSyncForGetTest.java    | 393 ++++++++++++++++++
 .../IgniteCacheExpiryPolicyAbstractTest.java  |   2 +-
 .../hashmap/GridHashMapLoadTest.java          |   3 +-
 .../testsuites/IgniteCacheTestSuite2.java     |   7 +-
 .../cache/ttl/CacheTtlAbstractSelfTest.java   |   2 +-
 39 files changed, 1262 insertions(+), 530 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
index 8d3ab74a0dee0..9d8a93f86b135 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
@@ -56,6 +56,7 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheSqlMetadata;
 import org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandler;
 import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.lang.GridTuple3;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.F;
@@ -702,6 +703,8 @@ public void testPutWithExpiration() throws Exception {
 
         Thread.sleep(2100);
 
+        waitExpired("putKey");
+
         assertNull(jcache().get("putKey"));
     }
 
@@ -733,6 +736,8 @@ public void testAddWithExpiration() throws Exception {
 
         Thread.sleep(2100);
 
+        waitExpired("addKey");
+
         assertNull(jcache().get("addKey"));
     }
 
@@ -865,6 +870,8 @@ public void testReplaceWithExpiration() throws Exception {
         // Use larger value to avoid false positives.
         Thread.sleep(2100);
 
+        waitExpired("replaceKey");
+
         assertNull(jcache().get("replaceKey"));
     }
 
@@ -2270,6 +2277,18 @@ private static String concat(Object[] vals, String delim) {
         }
     }
 
+    /**
+     * @param key Key.
+     * @throws Exception Exception.
+     */
+    private void waitExpired(final Object key) throws Exception {
+        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return jcache().get(key) == null;
+            }
+        }, 5000);
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 8c3c5d16bd514..aaea4f08eacb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -973,8 +973,8 @@ public GridCacheEntryEx entryEx(KeyCacheObject key, AffinityTopologyVersion topV
             cur = map.putEntryIfObsoleteOrAbsent(
                 topVer,
                 key,
-                null,
-                create, touch);
+                create,
+                touch);
         }
 
         return cur;
@@ -1908,61 +1908,105 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
 
                 Map<KeyCacheObject, EntryGetResult> misses = null;
 
+                boolean offheapRead = ctx.offheapRead(expiry, readerArgs != null);
+
                 for (KeyCacheObject key : keys) {
                     while (true) {
-                        GridCacheEntryEx entry = needEntry ? entryEx(key) : peekEx(key);
-
-                        if (entry == null) {
-                            if (!skipVals && ctx.config().isStatisticsEnabled())
-                                ctx.cache().metrics0().onRead(false);
-
-                            break;
-                        }
-
                         try {
-                            EntryGetResult res;
+                            EntryGetResult res = null;
 
                             boolean evt = !skipVals;
                             boolean updateMetrics = !skipVals;
 
-                            if (storeEnabled) {
-                                res = entry.innerGetAndReserveForLoad(ctx.isSwapOrOffheapEnabled(),
-                                    updateMetrics,
-                                    evt,
-                                    subjId,
-                                    taskName,
-                                    expiry,
-                                    !deserializeBinary,
-                                    readerArgs);
+                            GridCacheEntryEx entry = null;
+
+                            boolean skipEntry = false;
 
-                                assert res != null;
+                            if (offheapRead) {
+                                GridCacheSwapEntry swapEntry = ctx.swap().readSwapEntry(key);
 
-                                if (res.value() == null) {
-                                    if (misses == null)
-                                        misses = new HashMap<>();
+                                if (swapEntry != null) {
+                                    long expireTime = swapEntry.expireTime();
 
-                                    misses.put(key, res);
+                                    if (expireTime != 0) {
+                                        if (expireTime  >  U.currentTimeMillis()) {
+                                            res = new EntryGetWithTtlResult(swapEntry.value(),
+                                                swapEntry.version(),
+                                                false,
+                                                expireTime,
+                                                swapEntry.ttl());
+                                        }
+                                    }
+                                    else
+                                        res = new EntryGetResult(swapEntry.value(), swapEntry.version(), false);
+                                }
+
+                                if (res != null) {
+                                    skipEntry = true;
 
-                                    res = null;
+                                    if (evt) {
+                                        ctx.events().readEvent(key,
+                                            null,
+                                            swapEntry.value(),
+                                            subjId,
+                                            taskName,
+                                            !deserializeBinary);
+                                    }
+
+                                    if (updateMetrics && ctx.cache().configuration().isStatisticsEnabled())
+                                        ctx.cache().metrics0().onRead(true);
                                 }
                             }
-                            else {
-                                res = entry.innerGetVersioned(
-                                    null,
-                                    null,
-                                    ctx.isSwapOrOffheapEnabled(),
-                                    /*unmarshal*/true,
-                                    updateMetrics,
-                                    evt,
-                                    subjId,
-                                    null,
-                                    taskName,
-                                    expiry,
-                                    !deserializeBinary,
-                                    readerArgs);
-
-                                if (res == null)
-                                    ctx.evicts().touch(entry, topVer);
+
+                            if (!skipEntry) {
+                                entry = needEntry ? entryEx(key) : peekEx(key);
+
+                                if (entry == null) {
+                                    if (!skipVals && ctx.config().isStatisticsEnabled())
+                                        ctx.cache().metrics0().onRead(false);
+
+                                    break;
+                                }
+
+                                if (storeEnabled) {
+                                    res = entry.innerGetAndReserveForLoad(ctx.isSwapOrOffheapEnabled(),
+                                        updateMetrics,
+                                        evt,
+                                        subjId,
+                                        taskName,
+                                        expiry,
+                                        !deserializeBinary,
+                                        readerArgs);
+
+                                    assert res != null;
+
+                                    if (res.value() == null) {
+                                        if (misses == null)
+                                            misses = new HashMap<>();
+
+                                        misses.put(key, res);
+
+                                        res = null;
+                                    }
+                                }
+                                else {
+                                    res = entry.innerGetVersioned(
+                                        null,
+                                        null,
+                                        ctx.isSwapOrOffheapEnabled(),
+                                        /*unmarshal*/true,
+                                        updateMetrics,
+                                        evt,
+                                        subjId,
+                                        null,
+                                        taskName,
+                                        expiry,
+                                        !deserializeBinary,
+                                        readerArgs);
+
+                                    if (res == null)
+                                        ctx.evicts().touch(entry, topVer);
+                                }
                             }
 
                             if (res != null) {
@@ -1975,7 +2019,7 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                     true,
                                     needVer);
 
-                                if (tx == null || (!tx.implicit() && tx.isolation() == READ_COMMITTED))
+                                if (entry != null && (tx == null || (!tx.implicit() && tx.isolation() == READ_COMMITTED)))
                                     ctx.evicts().touch(entry, topVer);
 
                                 if (keysSize == 1)
@@ -5773,6 +5817,10 @@ protected abstract static class CacheExpiryPolicy implements IgniteCacheExpiryPo
                     return CU.toTtl(expiryPlc.getExpiryForAccess());
                 }
 
+                @Override public boolean hasAccessTtl() {
+                    return CU.toTtl(expiryPlc.getExpiryForAccess()) != CU.TTL_NOT_CHANGED;
+                }
+
                 @Override public long forCreate() {
                     return CU.toTtl(expiryPlc.getExpiryForCreation());
                 }
@@ -5801,6 +5849,10 @@ protected abstract static class CacheExpiryPolicy implements IgniteCacheExpiryPo
                     return accessTtl;
                 }
 
+                @Override public boolean hasAccessTtl() {
+                    return accessTtl != CU.TTL_NOT_CHANGED;
+                }
+
                 /** {@inheritDoc} */
                 @Override public long forUpdate() {
                     return CU.TTL_NOT_CHANGED;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheConcurrentMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheConcurrentMap.java
index e733114f4d95b..0b6d5ab4d71e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheConcurrentMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheConcurrentMap.java
@@ -38,7 +38,6 @@ public interface GridCacheConcurrentMap {
     /**
      * @param topVer Topology version.
      * @param key Key.
-     * @param val Value.
      * @param create Create flag.
      * @return Triple where the first element is current entry associated with the key,
      *      the second is created entry and the third is doomed (all may be null).
@@ -46,7 +45,6 @@ public interface GridCacheConcurrentMap {
     @Nullable public GridCacheMapEntry putEntryIfObsoleteOrAbsent(
         AffinityTopologyVersion topVer,
         KeyCacheObject key,
-        @Nullable CacheObject val,
         boolean create,
         boolean touch);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheConcurrentMapImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheConcurrentMapImpl.java
index 48dae76fca730..3830d476bb4ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheConcurrentMapImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheConcurrentMapImpl.java
@@ -110,8 +110,7 @@ public GridCacheConcurrentMapImpl(
 
     /** {@inheritDoc} */
     @Nullable @Override public GridCacheMapEntry putEntryIfObsoleteOrAbsent(final AffinityTopologyVersion topVer,
-        KeyCacheObject key, @Nullable final CacheObject val, final boolean create, final boolean touch) {
-
+        KeyCacheObject key, final boolean create, final boolean touch) {
         GridCacheMapEntry cur = null;
         GridCacheMapEntry created = null;
         GridCacheMapEntry created0 = null;
@@ -127,7 +126,7 @@ public GridCacheConcurrentMapImpl(
             if (entry == null) {
                 if (create) {
                     if (created0 == null)
-                        created0 = factory.create(ctx, topVer, key, key.hashCode(), val);
+                        created0 = factory.create(ctx, topVer, key, key.hashCode());
 
                     cur = created = created0;
 
@@ -142,7 +141,7 @@ public GridCacheConcurrentMapImpl(
 
                     if (create) {
                         if (created0 == null)
-                            created0 = factory.create(ctx, topVer, key, key.hashCode(), val);
+                            created0 = factory.create(ctx, topVer, key, key.hashCode());
 
                         cur = created = created0;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 3b44b5096f0b0..184f467d261c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -2057,6 +2057,21 @@ public Collection<KeyCacheObject> cacheKeysView(Collection<?> keys) {
         });
     }
 
+    /**
+     * Checks if it is possible to directly read offheap memory without entry creation (this
+     * is optimization to avoid unnecessary blocking synchronization on cache entry).
+     *
+     * @param expiryPlc Expiry policy for read operation.
+     * @param readers {@code True} if need update entry readers.
+     * @return {@code True} if it is possible directly read offheap instead of using {@link GridCacheEntryEx#innerGet}.
+     */
+    public boolean offheapRead(IgniteCacheExpiryPolicy expiryPlc, boolean readers) {
+        return offheapTiered() &&
+            isSwapOrOffheapEnabled() &&
+            (expiryPlc == null || !expiryPlc.hasAccessTtl()) &&
+            !readers;
+    }
+
     /**
      * @param part Partition.
      * @param affNodes Affinity nodes.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java
index 1c1873814e675..7a417d5bbca9f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_READ;
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_UNLOADED;
 import static org.apache.ignite.events.EventType.EVT_CACHE_STARTED;
 import static org.apache.ignite.events.EventType.EVT_CACHE_STOPPED;
@@ -61,6 +62,37 @@ public void removeListener(GridLocalEventListener lsnr) {
         cctx.gridEvents().removeLocalEventListener(lsnr);
     }
 
+    /**
+     * @param key Key for event.
+     * @param tx Possible surrounding transaction.
+     * @param val Read value.
+     * @param subjId Subject ID.
+     * @param taskName Task name.
+     * @param keepBinary Keep binary flag.
+     */
+    public void readEvent(KeyCacheObject key,
+        @Nullable IgniteInternalTx tx,
+        @Nullable CacheObject val,
+        @Nullable UUID subjId,
+        @Nullable String taskName,
+        boolean keepBinary) {
+        if (isRecordable(EVT_CACHE_OBJECT_READ)) {
+            addEvent(cctx.affinity().partition(key),
+                key,
+                tx,
+                null,
+                EVT_CACHE_OBJECT_READ,
+                val,
+                val != null,
+                val,
+                val != null,
+                subjId,
+                null,
+                taskName,
+                keepBinary);
+        }
+    }
+
     /**
      * @param part Partition.
      * @param key Key for the event.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 2237e227b60e3..d6d81de9d19a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -172,13 +172,11 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
      * @param cctx Cache context.
      * @param key Cache key.
      * @param hash Key hash value.
-     * @param val Entry value.
      */
     protected GridCacheMapEntry(
         GridCacheContext<?, ?> cctx,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
         if (log == null)
             log = U.logger(cctx.kernalContext(), logRef, GridCacheMapEntry.class);
@@ -191,12 +189,6 @@ protected GridCacheMapEntry(
         this.hash = hash;
         this.cctx = cctx;
 
-        val = cctx.kernalContext().cacheObjects().prepareForCache(val, cctx);
-
-        synchronized (this) {
-            value(val);
-        }
-
         ver = cctx.versions().next();
 
         startVer = ver.order();
@@ -388,7 +380,7 @@ protected CacheObject valueBytesUnlocked() {
     /**
      * @return {@code True} if start version.
      */
-    public boolean isStartVersion() {
+    private boolean isStartVersion() {
         return ver.nodeOrder() == cctx.localNode().order() && ver.order() == startVer;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntryFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntryFactory.java
index 4ee938532c58c..0b49f791da5ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntryFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntryFactory.java
@@ -28,14 +28,12 @@ public interface GridCacheMapEntryFactory {
      * @param topVer Topology version.
      * @param key Cache key.
      * @param hash Key hash value.
-     * @param val Entry value.
      * @return New cache entry.
      */
     public GridCacheMapEntry create(
         GridCacheContext ctx,
         AffinityTopologyVersion topVer,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     );
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java
index 159b3b85c9fbc..67c00dc99bd64 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSwapManager.java
@@ -25,6 +25,7 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Set;
@@ -162,6 +163,8 @@ public void unwindOffheapEvicts() {
         Collection<IgniteBiTuple<byte[], byte[]>> evicts = offheapEvicts.get();
 
         if (evicts != null) {
+            List<GridCacheEntryEx> entries = null;
+
             GridCacheVersion obsoleteVer = cctx.versions().next();
 
             for (IgniteBiTuple<byte[], byte[]> t : evicts) {
@@ -179,6 +182,12 @@ public void unwindOffheapEvicts() {
                         try {
                             if (entry.onOffheapEvict(vb, evictVer, obsoleteVer))
                                 cctx.cache().removeEntry(entry);
+                            else {
+                                if (entries == null)
+                                    entries = new ArrayList<>();
+
+                                entries.add(entry);
+                            }
 
                             break;
                         }
@@ -196,6 +205,11 @@ public void unwindOffheapEvicts() {
             }
 
             offheapEvicts.set(null);
+
+            if (entries != null) {
+                for (int i = 0; i < entries.size(); i++)
+                    cctx.evicts().touch(entries.get(i), AffinityTopologyVersion.NONE);
+            }
         }
     }
 
@@ -818,6 +832,28 @@ public boolean containsKey(KeyCacheObject key, int part) throws IgniteCheckedExc
             readOffheap, readSwap, valOnly);
     }
 
+    /**
+     * @param key Key.
+     * @return Read value.
+     * @throws IgniteCheckedException If failed.
+     */
+    @Nullable public GridCacheSwapEntry readSwapEntry(KeyCacheObject key) throws IgniteCheckedException {
+        assert offheapEnabled || swapEnabled;
+
+        GridCacheSwapEntry entry = read(key,
+            key.valueBytes(cctx.cacheObjectContext()),
+            cctx.affinity().partition(key),
+            false,
+            true,
+            true,
+            false);
+
+        assert entry == null || entry.value() != null : entry;
+        assert entry == null || entry.version() != null : entry;
+
+        return entry;
+    }
+
     /**
      * @param entry Entry to read.
      * @return Read value address.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridNoStorageCacheMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridNoStorageCacheMap.java
index 253288220d030..17bdfeea63a6d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridNoStorageCacheMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridNoStorageCacheMap.java
@@ -46,11 +46,11 @@ public GridNoStorageCacheMap(GridCacheContext ctx) {
 
     /** {@inheritDoc} */
     @Override public GridCacheMapEntry putEntryIfObsoleteOrAbsent(AffinityTopologyVersion topVer, KeyCacheObject key,
-        @Nullable CacheObject val, boolean create, boolean touch) {
+        boolean create, boolean touch) {
         if (create)
             return ctx.useOffheapEntry() ?
-                new GridDhtOffHeapCacheEntry(ctx, topVer, key, key.hashCode(), val) :
-                new GridDhtCacheEntry(ctx, topVer, key, key.hashCode(), val);
+                new GridDhtOffHeapCacheEntry(ctx, topVer, key, key.hashCode()) :
+                new GridDhtCacheEntry(ctx, topVer, key, key.hashCode());
         else
             return null;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheExpiryPolicy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheExpiryPolicy.java
index f82c5f0289132..96f1c6fa693a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheExpiryPolicy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheExpiryPolicy.java
@@ -45,6 +45,11 @@ public interface IgniteCacheExpiryPolicy {
      */
     public long forAccess();
 
+    /**
+     * @return {@code True} if expiry policy change ttl on entry read.
+     */
+    public boolean hasAccessTtl();
+
     /**
      * Callback for ttl update on entry access.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java
index f518934500144..28cd2029245c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java
@@ -50,15 +50,13 @@ public class GridDistributedCacheEntry extends GridCacheMapEntry {
      * @param ctx Cache context.
      * @param key Cache key.
      * @param hash Key hash value.
-     * @param val Entry value.
      */
     public GridDistributedCacheEntry(
         GridCacheContext ctx,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
-        super(ctx, key, hash, val);
+        super(ctx, key, hash);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedConcurrentMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedConcurrentMap.java
index cfbe9bbe782af..d59bf2dc5d41a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedConcurrentMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedConcurrentMap.java
@@ -80,13 +80,13 @@ public GridCachePartitionedConcurrentMap(GridCacheContext ctx) {
 
     /** {@inheritDoc} */
     @Override public GridCacheMapEntry putEntryIfObsoleteOrAbsent(AffinityTopologyVersion topVer, KeyCacheObject key,
-        @Nullable CacheObject val, boolean create, boolean touch) {
+        boolean create, boolean touch) {
         GridDhtLocalPartition part = localPartition(key, topVer, create);
 
         if (part == null)
             return null;
 
-        return part.putEntryIfObsoleteOrAbsent(topVer, key, val, create, touch);
+        return part.putEntryIfObsoleteOrAbsent(topVer, key, create, touch);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index a8cc599d50f2c..f8932867552bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -247,13 +247,12 @@ protected GridCacheMapEntryFactory entryFactory() {
                 GridCacheContext ctx,
                 AffinityTopologyVersion topVer,
                 KeyCacheObject key,
-                int hash,
-                CacheObject val
+                int hash
             ) {
                 if (ctx.useOffheapEntry())
-                    return new GridDhtOffHeapCacheEntry(ctx, topVer, key, hash, val);
+                    return new GridDhtOffHeapCacheEntry(ctx, topVer, key, hash);
 
-                return new GridDhtCacheEntry(ctx, topVer, key, hash, val);
+                return new GridDhtCacheEntry(ctx, topVer, key, hash);
             }
         };
     }
@@ -441,7 +440,7 @@ public GridDhtCacheEntry entryExx(KeyCacheObject key,
      * @return Cache entry.
      */
     protected GridDistributedCacheEntry createEntry(KeyCacheObject key) {
-        return new GridDhtDetachedCacheEntry(ctx, key, key.hashCode(), null, null, 0);
+        return new GridDhtDetachedCacheEntry(ctx, key, key.hashCode());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
index 39571ff4720e6..44720500de105 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
@@ -77,16 +77,14 @@ public class GridDhtCacheEntry extends GridDistributedCacheEntry {
      * @param topVer Topology version at the time of creation (if negative, then latest topology is assumed).
      * @param key Cache key.
      * @param hash Key hash value.
-     * @param val Entry value.
      */
     public GridDhtCacheEntry(
         GridCacheContext ctx,
         AffinityTopologyVersion topVer,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
-        super(ctx, key, hash, val);
+        super(ctx, key, hash);
 
         // Record this entry with partition.
         int p = cctx.affinity().partition(key);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 9f8498ad357f7..4475d48b2ac73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -281,8 +281,8 @@ public boolean valid() {
     /** {@inheritDoc} */
     @Override public GridCacheMapEntry putEntryIfObsoleteOrAbsent(
         AffinityTopologyVersion topVer, KeyCacheObject key,
-        @Nullable CacheObject val, boolean create, boolean touch) {
-        return map.putEntryIfObsoleteOrAbsent(topVer, key, val, create, touch);
+        boolean create, boolean touch) {
+        return map.putEntryIfObsoleteOrAbsent(topVer, key, create, touch);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtOffHeapCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtOffHeapCacheEntry.java
index 75ee45cb4798a..043b78f13dc08 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtOffHeapCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtOffHeapCacheEntry.java
@@ -35,16 +35,14 @@ public class GridDhtOffHeapCacheEntry extends GridDhtCacheEntry {
      * @param topVer Topology version at the time of creation (if negative, then latest topology is assumed).
      * @param key    Cache key.
      * @param hash   Key hash value.
-     * @param val    Entry value.
      */
     public GridDhtOffHeapCacheEntry(
         GridCacheContext ctx,
         AffinityTopologyVersion topVer,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
-        super(ctx, topVer, key, hash, val);
+        super(ctx, topVer, key, hash);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 519239aecb532..6f3f776e57db8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
 import org.apache.ignite.internal.processors.cache.GridCacheMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheSwapEntry;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest;
@@ -437,81 +438,114 @@ private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
 
         GridDhtCacheAdapter<K, V> cache = cache();
 
-        while (true) {
-            GridCacheEntryEx entry;
+        boolean offheapRead = cctx.offheapRead(expiryPlc, false);
+        boolean evt = !skipVals;
 
+        while (true) {
             try {
-                entry = cache.context().isSwapOrOffheapEnabled() ? cache.entryEx(key) : cache.peekEx(key);
-
-                // If our DHT cache do has value, then we peek it.
-                if (entry != null) {
-                    boolean isNew = entry.isNewLocked();
-
-                    EntryGetResult getRes = null;
-                    CacheObject v = null;
-                    GridCacheVersion ver = null;
-
-                    if (needVer) {
-                        getRes = entry.innerGetVersioned(
-                            null,
-                            null,
-                            /*swap*/true,
-                            /*unmarshal*/true,
-                            /**update-metrics*/false,
-                            /*event*/!skipVals,
-                            subjId,
-                            null,
-                            taskName,
-                            expiryPlc,
-                            !deserializeBinary,
-                            null);
-
-                        if (getRes != null) {
-                            v = getRes.value();
-                            ver = getRes.version();
+                boolean skipEntry = false;
+
+                EntryGetResult getRes = null;
+                CacheObject v = null;
+                GridCacheVersion ver = null;
+
+                if (offheapRead) {
+                    GridCacheSwapEntry swapEntry = cctx.swap().readSwapEntry(key);
+
+                    if (swapEntry != null) {
+                        long expireTime = swapEntry.expireTime();
+
+                        if (expireTime == 0 || expireTime > U.currentTimeMillis()) {
+                            skipEntry = true;
+
+                            v = swapEntry.value();
+
+                            if (needVer)
+                                ver = swapEntry.version();
+
+                            if (evt) {
+                                cctx.events().readEvent(key,
+                                    null,
+                                    swapEntry.value(),
+                                    subjId,
+                                    taskName,
+                                    !deserializeBinary);
+                            }
                         }
                     }
-                    else {
-                        v = entry.innerGet(
-                            null,
-                            null,
-                            /*swap*/true,
-                            /*read-through*/false,
-                            /**update-metrics*/false,
-                            /*event*/!skipVals,
-                            /*temporary*/false,
-                            subjId,
-                            null,
-                            taskName,
-                            expiryPlc,
-                            !deserializeBinary);
-                    }
+                }
+
+                if (!skipEntry) {
+                    GridCacheEntryEx entry =
+                        cache.context().isSwapOrOffheapEnabled() ? cache.entryEx(key) : cache.peekEx(key);
+
+                    // If our DHT cache do has value, then we peek it.
+                    if (entry != null) {
+                        boolean isNew = entry.isNewLocked();
+
+                        if (needVer) {
+                            getRes = entry.innerGetVersioned(
+                                null,
+                                null,
+                                /*swap*/true,
+                                /*unmarshal*/true,
+                                /*update-metrics*/false,
+                                /*event*/evt,
+                                subjId,
+                                null,
+                                taskName,
+                                expiryPlc,
+                                !deserializeBinary,
+                                null);
+
+                            if (getRes != null) {
+                                v = getRes.value();
+                                ver = getRes.version();
+                            }
+                        }
+                        else {
+                            v = entry.innerGet(
+                                null,
+                                null,
+                                /*swap*/true,
+                                /*read-through*/false,
+                                /*update-metrics*/false,
+                                /*event*/evt,
+                                /*temporary*/false,
+                                subjId,
+                                null,
+                                taskName,
+                                expiryPlc,
+                                !deserializeBinary);
+                        }
 
-                    cache.context().evicts().touch(entry, topVer);
+                        cache.context().evicts().touch(entry, topVer);
 
-                    // Entry was not in memory or in swap, so we remove it from cache.
-                    if (v == null) {
-                        if (isNew && entry.markObsoleteIfEmpty(ver))
-                            cache.removeEntry(entry);
-                    }
-                    else {
-                        cctx.addResult(locVals,
-                            key,
-                            v,
-                            skipVals,
-                            keepCacheObjects,
-                            deserializeBinary,
-                            true,
-                            getRes,
-                            ver,
-                            0,
-                            0,
-                            needVer);
-
-                        return true;
+                        // Entry was not in memory or in swap, so we remove it from cache.
+                        if (v == null) {
+                            if (isNew && entry.markObsoleteIfEmpty(ver))
+                                cache.removeEntry(entry);
+                        }
                     }
                 }
 
+                if (v != null) {
+                    cctx.addResult(locVals,
+                        key,
+                        v,
+                        skipVals,
+                        keepCacheObjects,
+                        deserializeBinary,
+                        true,
+                        getRes,
+                        ver,
+                        0,
+                        0,
+                        needVer);
+
+                    return true;
+                }
+
                 boolean topStable = cctx.isReplicated() || topVer.equals(cctx.topology().topologyVersion());
 
                 // Entry not found, do not continue search if topology did not change and there is no store.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index a3f6b72376e8f..85ccd6b86d334 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
 import org.apache.ignite.internal.processors.cache.GridCacheFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheSwapEntry;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.near.CacheVersionedValue;
@@ -360,74 +361,108 @@ private boolean localGet(AffinityTopologyVersion topVer, int part) {
 
         GridDhtCacheAdapter colocated = cctx.dht();
 
-        while (true) {
-            GridCacheEntryEx entry;
+        boolean offheapRead = cctx.offheapRead(expiryPlc, false);
+        boolean evt = !skipVals;
 
+        while (true) {
             try {
-                entry = colocated.context().isSwapOrOffheapEnabled() ? colocated.entryEx(key) :
-                    colocated.peekEx(key);
-
-                // If our DHT cache do has value, then we peek it.
-                if (entry != null) {
-                    boolean isNew = entry.isNewLocked();
-
-                    CacheObject v = null;
-                    GridCacheVersion ver = null;
-
-                    if (needVer) {
-                        EntryGetResult res = entry.innerGetVersioned(
-                            null,
-                            null,
-                            /*swap*/true,
-                            /*unmarshal*/true,
-                            /**update-metrics*/false,
-                            /*event*/!skipVals,
-                            subjId,
-                            null,
-                            taskName,
-                            expiryPlc,
-                            true,
-                            null);
-
-                        if (res != null) {
-                            v = res.value();
-                            ver = res.version();
+                CacheObject v = null;
+                GridCacheVersion ver = null;
+
+                boolean skipEntry = false;
+
+                if (offheapRead) {
+                    GridCacheSwapEntry swapEntry = cctx.swap().readSwapEntry(key);
+
+                    if (swapEntry != null) {
+                        long expireTime = swapEntry.expireTime();
+
+                        if (expireTime == 0 || expireTime > U.currentTimeMillis()) {
+                            skipEntry = true;
+
+                            v = swapEntry.value();
+
+                            if (needVer)
+                                ver = swapEntry.version();
+
+                            if (evt) {
+                                cctx.events().readEvent(key,
+                                    null,
+                                    swapEntry.value(),
+                                    subjId,
+                                    taskName,
+                                    !deserializeBinary);
+                            }
                         }
+                        else
+                            skipEntry = false;
                     }
-                    else {
-                        v = entry.innerGet(
-                            null,
-                            null,
-                            /*swap*/true,
-                            /*read-through*/false,
-                            /**update-metrics*/false,
-                            /*event*/!skipVals,
-                            /*temporary*/false,
-                            subjId,
-                            null,
-                            taskName,
-                            expiryPlc,
-                            true);
-                    }
+                }
 
-                    colocated.context().evicts().touch(entry, topVer);
+                if (!skipEntry) {
+                    GridCacheEntryEx entry = colocated.context().isSwapOrOffheapEnabled() ? colocated.entryEx(key) :
+                        colocated.peekEx(key);
+
+                    // If our DHT cache do has value, then we peek it.
+                    if (entry != null) {
+                        boolean isNew = entry.isNewLocked();
+
+                        if (needVer) {
+                            EntryGetResult res = entry.innerGetVersioned(
+                                null,
+                                null,
+                                /*swap*/true,
+                                /*unmarshal*/true,
+                                /*update-metrics*/false,
+                                /*event*/evt,
+                                subjId,
+                                null,
+                                taskName,
+                                expiryPlc,
+                                true,
+                                null);
+
+                            if (res != null) {
+                                v = res.value();
+                                ver = res.version();
+                            }
+                        }
+                        else {
+                            v = entry.innerGet(
+                                null,
+                                null,
+                                /*swap*/true,
+                                /*read-through*/false,
+                                /*update-metrics*/false,
+                                /*event*/evt,
+                                /*temporary*/false,
+                                subjId,
+                                null,
+                                taskName,
+                                expiryPlc,
+                                true);
+                        }
+
+                        colocated.context().evicts().touch(entry, topVer);
 
-                    // Entry was not in memory or in swap, so we remove it from cache.
-                    if (v == null) {
-                        if (isNew && entry.markObsoleteIfEmpty(ver))
-                            colocated.removeEntry(entry);
+                        if (v == null) {
+                            // Entry was not in memory or in swap, so we remove it from cache.
+                            if (isNew && entry.markObsoleteIfEmpty(ver))
+                                colocated.removeEntry(entry);
+                        }
                     }
-                    else {
-                        if (!skipVals && cctx.config().isStatisticsEnabled())
-                            cctx.cache().metrics0().onRead(true);
+                }
 
-                        if (!skipVals)
-                            setResult(v, ver);
-                        else
-                            setSkipValueResult(true, ver);
+                if (v != null) {
+                    if (!skipVals && cctx.config().isStatisticsEnabled())
+                        cctx.cache().metrics0().onRead(true);
 
-                        return true;
-                    }
+                    if (!skipVals)
+                        setResult(v, ver);
+                    else
+                        setSkipValueResult(true, ver);
+
+                    return true;
                 }
 
                 boolean topStable = cctx.isReplicated() || topVer.equals(cctx.topology().topologyVersion());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 4159359a2f780..5497980efe9d6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMapEntryFactory;
 import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.GridCacheReturn;
+import org.apache.ignite.internal.processors.cache.GridCacheSwapEntry;
 import org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult;
 import org.apache.ignite.internal.processors.cache.GridDeferredAckMessageSender;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
@@ -195,13 +196,12 @@ public GridDhtAtomicCache(GridCacheContext<K, V> ctx, GridCacheConcurrentMap map
                 GridCacheContext ctx,
                 AffinityTopologyVersion topVer,
                 KeyCacheObject key,
-                int hash,
-                CacheObject val
+                int hash
             ) {
                 if (ctx.useOffheapEntry())
-                    return new GridDhtAtomicOffHeapCacheEntry(ctx, topVer, key, hash, val);
+                    return new GridDhtAtomicOffHeapCacheEntry(ctx, topVer, key, hash);
 
-                return new GridDhtAtomicCacheEntry(ctx, topVer, key, hash, val);
+                return new GridDhtAtomicCacheEntry(ctx, topVer, key, hash);
             }
         };
     }
@@ -1566,114 +1566,165 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
 
         final IgniteCacheExpiryPolicy expiry = skipVals ? null : expiryPolicy(expiryPlc);
 
+        boolean evt = !skipVals;
+
         // Optimisation: try to resolve value locally and escape 'get future' creation.
         if (!forcePrimary && ctx.affinityNode()) {
-            Map<K, V> locVals = U.newHashMap(keys.size());
-
-            boolean success = true;
-
-            // Optimistically expect that all keys are available locally (avoid creation of get future).
-            for (KeyCacheObject key : keys) {
-                GridCacheEntryEx entry = null;
-
-                while (true) {
-                    try {
-                        entry = ctx.isSwapOrOffheapEnabled() ? entryEx(key) : peekEx(key);
-
-                        // If our DHT cache do has value, then we peek it.
-                        if (entry != null) {
-                            boolean isNew = entry.isNewLocked();
-
-                            EntryGetResult getRes = null;
-                            CacheObject v = null;
-                            GridCacheVersion ver = null;
-
-                            if (needVer) {
-                                getRes = entry.innerGetVersioned(
-                                    null,
-                                    null,
-                                    /*swap*/true,
-                                    /*unmarshal*/true,
-                                    /**update-metrics*/false,
-                                    /*event*/!skipVals,
-                                    subjId,
-                                    null,
-                                    taskName,
-                                    expiry,
+            try {
+                Map<K, V> locVals = U.newHashMap(keys.size());
+
+                boolean success = true;
+                boolean offheapRead = ctx.offheapRead(expiry, false);
+
+                // Optimistically expect that all keys are available locally (avoid creation of get future).
+                for (KeyCacheObject key : keys) {
+                    if (offheapRead) {
+                        GridCacheSwapEntry swapEntry = ctx.swap().readSwapEntry(key);
+
+                        if (swapEntry != null) {
+                            long expireTime = swapEntry.expireTime();
+
+                            if (expireTime == 0 || expireTime > U.currentTimeMillis()) {
+                                ctx.addResult(locVals,
+                                    key,
+                                    swapEntry.value(),
+                                    skipVals,
+                                    false,
+                                    deserializeBinary,
                                     true,
-                                    null);
-
-                                if (getRes != null) {
-                                    v = getRes.value();
-                                    ver = getRes.version();
-                                }
-                            }
-                            else {
-                                v = entry.innerGet(null,
-                                    null,
-                                    /*swap*/true,
-                                    /*read-through*/false,
-                                    /**update-metrics*/false,
-                                    /*event*/!skipVals,
-                                    /*temporary*/false,
-                                    subjId,
                                     null,
-                                    taskName,
-                                    expiry,
-                                    !deserializeBinary);
-                            }
-
-                            // Entry was not in memory or in swap, so we remove it from cache.
-                            if (v == null) {
-                                GridCacheVersion obsoleteVer = context().versions().next();
-
-                                if (isNew && entry.markObsoleteIfEmpty(obsoleteVer))
-                                    removeEntry(entry);
-
-                                success = false;
+                                    swapEntry.version(),
+                                    0,
+                                    0,
+                                    needVer);
+
+                                if (evt) {
+                                    ctx.events().readEvent(key,
+                                        null,
+                                        swapEntry.value(),
+                                        subjId,
+                                        taskName,
+                                        !deserializeBinary);
+                                }
                             }
                             else
-                                ctx.addResult(locVals, key, v, skipVals, false, deserializeBinary, true,
-                                    getRes, ver, 0, 0, needVer);
+                                success = false;
                         }
                         else
                             success = false;
-
-                        break; // While.
                     }
-                    catch (GridCacheEntryRemovedException ignored) {
-                        // No-op, retry.
-                    }
-                    catch (GridDhtInvalidPartitionException ignored) {
-                        success = false;
+                    else {
+                        GridCacheEntryEx entry = null;
+
+                        while (true) {
+                            try {
+                                entry = ctx.isSwapOrOffheapEnabled() ? entryEx(key) : peekEx(key);
+
+                                // If our DHT cache do has value, then we peek it.
+                                if (entry != null) {
+                                    boolean isNew = entry.isNewLocked();
+
+                                    EntryGetResult getRes = null;
+                                    CacheObject v = null;
+                                    GridCacheVersion ver = null;
+
+                                    if (needVer) {
+                                        getRes = entry.innerGetVersioned(
+                                            null,
+                                            null,
+                                            /*swap*/true,
+                                            /*unmarshal*/true,
+                                            /*update-metrics*/false,
+                                            /*event*/evt,
+                                            subjId,
+                                            null,
+                                            taskName,
+                                            expiry,
+                                            true,
+                                            null);
+
+                                        if (getRes != null) {
+                                            v = getRes.value();
+                                            ver = getRes.version();
+                                        }
+                                    }
+                                    else {
+                                        v = entry.innerGet(null,
+                                            null,
+                                            /*swap*/true,
+                                            /*read-through*/false,
+                                            /*update-metrics*/false,
+                                            /*event*/evt,
+                                            /*temporary*/false,
+                                            subjId,
+                                            null,
+                                            taskName,
+                                            expiry,
+                                            !deserializeBinary);
+                                    }
+
+                                    // Entry was not in memory or in swap, so we remove it from cache.
+                                    if (v == null) {
+                                        if (isNew && entry.markObsoleteIfEmpty(context().versions().next()))
+                                            removeEntry(entry);
+
+                                        success = false;
+                                    }
+                                    else {
+                                        ctx.addResult(locVals,
+                                            key,
+                                            v,
+                                            skipVals,
+                                            false,
+                                            deserializeBinary,
+                                            true,
+                                            getRes,
+                                            ver,
+                                            0,
+                                            0,
+                                            needVer);
+                                    }
+                                }
+                                else
+                                    success = false;
 
-                        break; // While.
-                    }
-                    catch (IgniteCheckedException e) {
-                        return new GridFinishedFuture<>(e);
-                    }
-                    finally {
-                        if (entry != null)
-                            ctx.evicts().touch(entry, topVer);
+                                break; // While.
+                            }
+                            catch (GridCacheEntryRemovedException ignored) {
+                                // No-op, retry.
+                            }
+                            catch (GridDhtInvalidPartitionException ignored) {
+                                success = false;
+
+                                break; // While.
+                            }
+                            finally {
+                                if (entry != null)
+                                    ctx.evicts().touch(entry, topVer);
+                            }
+                        }
+
+                        if (!success)
+                            break;
+                        else if (!skipVals && ctx.config().isStatisticsEnabled())
+                            metrics0().onRead(true);
                     }
                 }
 
-                if (!success)
-                    break;
-                else if (!skipVals && ctx.config().isStatisticsEnabled())
-                    metrics0().onRead(true);
-            }
+                if (success) {
+                    sendTtlUpdateRequest(expiry);
 
-            if (success) {
-                sendTtlUpdateRequest(expiry);
+                    return new GridFinishedFuture<>(locVals);
+                }
 
-                return new GridFinishedFuture<>(locVals);
+                if (expiry != null)
+                    expiry.reset();
+            }
+            catch (IgniteCheckedException e) {
+                return new GridFinishedFuture<>(e);
             }
         }
 
-        if (expiry != null)
-            expiry.reset();
-
         // Either reload or not all values are available locally.
         GridPartitionedGetFuture<K, V> fut = new GridPartitionedGetFuture<>(ctx,
             keys,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCacheEntry.java
index 3f014d54f6ef5..15278d7ffe85f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCacheEntry.java
@@ -34,16 +34,14 @@ public class GridDhtAtomicCacheEntry extends GridDhtCacheEntry {
      * @param topVer Topology version at the time of creation (if negative, then latest topology is assumed).
      * @param key Cache key.
      * @param hash Key hash value.
-     * @param val Entry value.
      */
     public GridDhtAtomicCacheEntry(
         GridCacheContext ctx,
         AffinityTopologyVersion topVer,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
-        super(ctx, topVer, key, hash, val);
+        super(ctx, topVer, key, hash);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicOffHeapCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicOffHeapCacheEntry.java
index c86d477b1924b..d00493df78c5c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicOffHeapCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicOffHeapCacheEntry.java
@@ -35,16 +35,14 @@ public class GridDhtAtomicOffHeapCacheEntry extends GridDhtAtomicCacheEntry {
      * @param topVer Topology version at the time of creation (if negative, then latest topology is assumed).
      * @param key    Cache key.
      * @param hash   Key hash value.
-     * @param val    Entry value.
      */
     public GridDhtAtomicOffHeapCacheEntry(
         GridCacheContext ctx,
         AffinityTopologyVersion topVer,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
-        super(ctx, topVer, key, hash, val);
+        super(ctx, topVer, key, hash);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index e1e0ec2bf087e..c09ad9b03ec2d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -42,6 +42,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMapEntryFactory;
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheReturn;
+import org.apache.ignite.internal.processors.cache.GridCacheSwapEntry;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
@@ -123,13 +124,12 @@ public GridDhtColocatedCache(GridCacheContext<K, V> ctx, GridCacheConcurrentMap
                 GridCacheContext ctx,
                 AffinityTopologyVersion topVer,
                 KeyCacheObject key,
-                int hash,
-                CacheObject val
+                int hash
             ) {
                 if (ctx.useOffheapEntry())
-                    return new GridDhtColocatedOffHeapCacheEntry(ctx, topVer, key, hash, val);
+                    return new GridDhtColocatedOffHeapCacheEntry(ctx, topVer, key, hash);
 
-                return new GridDhtColocatedCacheEntry(ctx, topVer, key, hash, val);
+                return new GridDhtColocatedCacheEntry(ctx, topVer, key, hash);
             }
         };
     }
@@ -452,121 +452,165 @@ public final IgniteInternalFuture<Map<K, V>> loadAsync(
 
         // Optimisation: try to resolve value locally and escape 'get future' creation.
         if (!forcePrimary) {
-            Map<K, V> locVals = null;
+            try {
+                Map<K, V> locVals = null;
 
-            boolean success = true;
+                boolean success = true;
+                boolean offheapRead = ctx.offheapRead(expiryPlc, false);
+                boolean evt = !skipVals;
 
-            // Optimistically expect that all keys are available locally (avoid creation of get future).
-            for (KeyCacheObject key : keys) {
-                GridCacheEntryEx entry = null;
-
-                while (true) {
-                    try {
-                        entry = ctx.isSwapOrOffheapEnabled() ? entryEx(key) : peekEx(key);
-
-                        // If our DHT cache do has value, then we peek it.
-                        if (entry != null) {
-                            boolean isNew = entry.isNewLocked();
-
-                            EntryGetResult getRes = null;
-                            CacheObject v = null;
-                            GridCacheVersion ver = null;
-
-                            if (needVer) {
-                                getRes = entry.innerGetVersioned(
-                                    null,
-                                    null,
-                                    /*swap*/true,
-                                    /*unmarshal*/true,
-                                    /**update-metrics*/false,
-                                    /*event*/!skipVals,
-                                    subjId,
-                                    null,
-                                    taskName,
-                                    expiryPlc,
-                                    !deserializeBinary,
-                                    null);
-
-                                if (getRes != null) {
-                                    v = getRes.value();
-                                    ver = getRes.version();
-                                }
-                            }
-                            else {
-                                v = entry.innerGet(
-                                    null,
-                                    null,
-                                    /*swap*/true,
-                                    /*read-through*/false,
-                                    /**update-metrics*/false,
-                                    /*event*/!skipVals,
-                                    /*temporary*/false,
-                                    subjId,
-                                    null,
-                                    taskName,
-                                    expiryPlc,
-                                    !deserializeBinary);
-                            }
-
-                            // Entry was not in memory or in swap, so we remove it from cache.
-                            if (v == null) {
-                                GridCacheVersion obsoleteVer = context().versions().next();
+                // Optimistically expect that all keys are available locally (avoid creation of get future).
+                for (KeyCacheObject key : keys) {
+                    if (offheapRead) {
+                        GridCacheSwapEntry swapEntry = ctx.swap().readSwapEntry(key);
 
-                                if (isNew && entry.markObsoleteIfEmpty(obsoleteVer))
-                                    removeEntry(entry);
+                        if (swapEntry != null) {
+                            long expireTime = swapEntry.expireTime();
 
-                                success = false;
-                            }
-                            else {
+                            if (expireTime == 0 || expireTime > U.currentTimeMillis()) {
                                 if (locVals == null)
                                     locVals = U.newHashMap(keys.size());
 
                                 ctx.addResult(locVals,
                                     key,
-                                    v,
+                                    swapEntry.value(),
                                     skipVals,
-                                    keepCacheObj,
+                                    false,
                                     deserializeBinary,
                                     true,
-                                    getRes,
-                                    ver,
+                                    null,
+                                    swapEntry.version(),
                                     0,
                                     0,
                                     needVer);
+
+                                if (evt) {
+                                    ctx.events().readEvent(key,
+                                        null,
+                                        swapEntry.value(),
+                                        subjId,
+                                        taskName,
+                                        !deserializeBinary);
+                                }
                             }
+                            else
+                                success = false;
                         }
                         else
                             success = false;
-
-                        break; // While.
-                    }
-                    catch (GridCacheEntryRemovedException ignored) {
-                        // No-op, retry.
                     }
-                    catch (GridDhtInvalidPartitionException ignored) {
-                        success = false;
+                    else {
+                        GridCacheEntryEx entry = null;
+
+                        while (true) {
+                            try {
+                                entry = ctx.isSwapOrOffheapEnabled() ? entryEx(key) : peekEx(key);
+
+                                // If our DHT cache do has value, then we peek it.
+                                if (entry != null) {
+                                    boolean isNew = entry.isNewLocked();
+
+                                    EntryGetResult getRes = null;
+                                    CacheObject v = null;
+                                    GridCacheVersion ver = null;
+
+                                    if (needVer) {
+                                        getRes = entry.innerGetVersioned(
+                                            null,
+                                            null,
+                                            /*swap*/true,
+                                            /*unmarshal*/true,
+                                            /*update-metrics*/false,
+                                            /*event*/evt,
+                                            subjId,
+                                            null,
+                                            taskName,
+                                            expiryPlc,
+                                            !deserializeBinary,
+                                            null);
+
+                                        if (getRes != null) {
+                                            v = getRes.value();
+                                            ver = getRes.version();
+                                        }
+                                    }
+                                    else {
+                                        v = entry.innerGet(
+                                            null,
+                                            null,
+                                            /*swap*/true,
+                                            /*read-through*/false,
+                                            /*update-metrics*/false,
+                                            /*event*/evt,
+                                            /*temporary*/false,
+                                            subjId,
+                                            null,
+                                            taskName,
+                                            expiryPlc,
+                                            !deserializeBinary);
+                                    }
+
+                                    // Entry was not in memory or in swap, so we remove it from cache.
+                                    if (v == null) {
+                                        GridCacheVersion obsoleteVer = context().versions().next();
+
+                                        if (isNew && entry.markObsoleteIfEmpty(obsoleteVer))
+                                            removeEntry(entry);
+
+                                        success = false;
+                                    }
+                                    else {
+                                        if (locVals == null)
+                                            locVals = U.newHashMap(keys.size());
+
+                                        ctx.addResult(locVals,
+                                            key,
+                                            v,
+                                            skipVals,
+                                            keepCacheObj,
+                                            deserializeBinary,
+                                            true,
+                                            getRes,
+                                            ver,
+                                            0,
+                                            0,
+                                            needVer);
+                                    }
+                                }
+                                else
+                                    success = false;
 
-                        break; // While.
-                    }
-                    catch (IgniteCheckedException e) {
-                        return new GridFinishedFuture<>(e);
-                    }
-                    finally {
-                        if (entry != null)
-                            context().evicts().touch(entry, topVer);
+                                break; // While.
+                            }
+                            catch (GridCacheEntryRemovedException ignored) {
+                                // No-op, retry.
+                            }
+                            catch (GridDhtInvalidPartitionException ignored) {
+                                success = false;
+
+                                break; // While.
+                            }
+                            finally {
+                                if (entry != null)
+                                    context().evicts().touch(entry, topVer);
+                            }
+                        }
                     }
-                }
 
-                if (!success)
-                    break;
-                else if (!skipVals && ctx.config().isStatisticsEnabled())
-                    ctx.cache().metrics0().onRead(true);
-            }
+                    if (!success)
+                        break;
+                    else if (!skipVals && ctx.config().isStatisticsEnabled())
+                        ctx.cache().metrics0().onRead(true);
+                }
 
-            if (success) {
-                sendTtlUpdateRequest(expiryPlc);
+                if (success) {
+                    sendTtlUpdateRequest(expiryPlc);
 
-                return new GridFinishedFuture<>(locVals);
+                    return new GridFinishedFuture<>(locVals);
+                }
+            }
+            catch (IgniteCheckedException e) {
+                return new GridFinishedFuture<>(e);
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCacheEntry.java
index cc71e115519b8..db837a74239ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCacheEntry.java
@@ -33,16 +33,14 @@ public class GridDhtColocatedCacheEntry extends GridDhtCacheEntry {
      * @param topVer Topology version at the time of creation (if negative, then latest topology is assumed).
      * @param key Cache key.
      * @param hash Key hash value.
-     * @param val Entry value.
      */
     public GridDhtColocatedCacheEntry(
         GridCacheContext ctx,
         AffinityTopologyVersion topVer,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
-        super(ctx, topVer, key, hash, val);
+        super(ctx, topVer, key, hash);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedOffHeapCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedOffHeapCacheEntry.java
index 8080ca391ebdf..a6869d7d19880 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedOffHeapCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedOffHeapCacheEntry.java
@@ -35,16 +35,14 @@ public class GridDhtColocatedOffHeapCacheEntry extends GridDhtColocatedCacheEntr
      * @param topVer Topology version at the time of creation (if negative, then latest topology is assumed).
      * @param key    Cache key.
      * @param hash   Key hash value.
-     * @param val    Entry value.
      */
     public GridDhtColocatedOffHeapCacheEntry(
         GridCacheContext ctx,
         AffinityTopologyVersion topVer,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
-        super(ctx, topVer, key, hash, val);
+        super(ctx, topVer, key, hash);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
index 2e055604b2687..3ae57f2b36d37 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
@@ -35,13 +35,9 @@ public class GridDhtDetachedCacheEntry extends GridDistributedCacheEntry {
      * @param ctx Cache context.
      * @param key Cache key.
      * @param hash Key hash value.
-     * @param val Entry value.
-     * @param next Next entry in the linked list.
-     * @param hdrId Header ID.
      */
-    public GridDhtDetachedCacheEntry(GridCacheContext ctx, KeyCacheObject key, int hash, CacheObject val,
-        GridCacheMapEntry next, int hdrId) {
-        super(ctx, key, hash, val);
+    public GridDhtDetachedCacheEntry(GridCacheContext ctx, KeyCacheObject key, int hash) {
+        super(ctx, key, hash);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
index 4ddad74157987..8cefe0c9287ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
@@ -96,15 +96,14 @@ protected GridNearCacheAdapter(GridCacheContext<K, V> ctx) {
                 GridCacheContext ctx,
                 AffinityTopologyVersion topVer,
                 KeyCacheObject key,
-                int hash,
-                CacheObject val
+                int hash
             ) {
                 // Can't hold any locks here - this method is invoked when
                 // holding write-lock on the whole cache map.
                 if (ctx.useOffheapEntry())
-                    return new GridNearOffHeapCacheEntry(ctx, key, hash, val);
+                    return new GridNearOffHeapCacheEntry(ctx, key, hash);
 
-                return new GridNearCacheEntry(ctx, key, hash, val);
+                return new GridNearCacheEntry(ctx, key, hash);
             }
         };
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index d022805a76bf2..5d449435f09bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -65,15 +65,13 @@ public class GridNearCacheEntry extends GridDistributedCacheEntry {
      * @param ctx Cache context.
      * @param key Cache key.
      * @param hash Key hash value.
-     * @param val Entry value.
      */
     public GridNearCacheEntry(
         GridCacheContext ctx,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
-        super(ctx, key, hash, val);
+        super(ctx, key, hash);
 
         part = ctx.affinity().partition(key);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOffHeapCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOffHeapCacheEntry.java
index 1558f4c9a18a7..f48256dcf1d90 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOffHeapCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOffHeapCacheEntry.java
@@ -32,15 +32,13 @@ public class GridNearOffHeapCacheEntry extends GridNearCacheEntry {
      * @param ctx   Cache context.
      * @param key   Cache key.
      * @param hash  Key hash value.
-     * @param val   Entry value.
      */
     public GridNearOffHeapCacheEntry(
         GridCacheContext ctx,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
-        super(ctx, key, hash, val);
+        super(ctx, key, hash);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
index 5b44d759f9080..e88742ca5b4ff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
@@ -86,10 +86,9 @@ public GridLocalCache(GridCacheContext<K, V> ctx) {
                 GridCacheContext ctx,
                 AffinityTopologyVersion topVer,
                 KeyCacheObject key,
-                int hash,
-                CacheObject val
+                int hash
             ) {
-                return new GridLocalCacheEntry(ctx, key, hash, val);
+                return new GridLocalCacheEntry(ctx, key, hash);
             }
         };
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java
index bc61333514ff8..6572672bf09fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java
@@ -42,15 +42,13 @@ public class GridLocalCacheEntry extends GridCacheMapEntry {
      * @param ctx  Cache registry.
      * @param key  Cache key.
      * @param hash Key hash value.
-     * @param val Entry value.
      */
     GridLocalCacheEntry(
         GridCacheContext ctx,
         KeyCacheObject key,
-        int hash,
-        CacheObject val
+        int hash
     ) {
-        super(ctx, key, hash, val);
+        super(ctx, key, hash);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index f86df2f29d684..b181e16220e37 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -22,7 +22,6 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -53,6 +52,7 @@
 import org.apache.ignite.internal.processors.cache.GridCachePreloader;
 import org.apache.ignite.internal.processors.cache.GridCachePreloaderAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheReturn;
+import org.apache.ignite.internal.processors.cache.GridCacheSwapEntry;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
@@ -389,7 +389,7 @@ private Map<K, V> getAllInternal(@Nullable Collection<? extends K> keys,
 
         UUID subjId = ctx.subjectIdPerCall(null, opCtx);
 
-        Map<K, V> vals = new HashMap<>(keys.size(), 1.0f);
+        Map<K, V> vals = U.newHashMap(keys.size());
 
         if (keyCheck)
             validateCacheKeys(keys);
@@ -397,97 +397,144 @@ private Map<K, V> getAllInternal(@Nullable Collection<? extends K> keys,
         final IgniteCacheExpiryPolicy expiry = expiryPolicy(opCtx != null ? opCtx.expiry() : null);
 
         boolean success = true;
+        final boolean offheapRead = ctx.offheapRead(expiry, false);
+        final boolean evt = !skipVals;
 
         for (K key : keys) {
             if (key == null)
                 throw new NullPointerException("Null key.");
 
-            GridCacheEntryEx entry = null;
-
             KeyCacheObject cacheKey = ctx.toCacheKeyObject(key);
 
-            while (true) {
-                try {
-                    entry = swapOrOffheap ? entryEx(cacheKey) : peekEx(cacheKey);
+            boolean skipEntry = false;
 
-                    if (entry != null) {
-                        CacheObject v;
+            if (offheapRead) {
+                GridCacheSwapEntry swapEntry = ctx.swap().readSwapEntry(cacheKey);
 
-                        if (needVer) {
-                            EntryGetResult res = entry.innerGetVersioned(
-                                null,
-                                null,
-                                /*swap*/swapOrOffheap,
-                                /*unmarshal*/true,
-                                /**update-metrics*/false,
-                                /*event*/!skipVals,
-                                subjId,
-                                null,
-                                taskName,
-                                expiry,
-                                !deserializeBinary,
-                                null);
-
-                            if (res != null) {
-                                ctx.addResult(
-                                    vals,
-                                    cacheKey,
-                                    res,
-                                    skipVals,
-                                    false,
-                                    deserializeBinary,
-                                    true,
-                                    needVer);
-                            }
-                            else
-                                success = false;
-                        }
-                        else {
-                            v = entry.innerGet(
-                                null,
+                if (swapEntry != null) {
+                    long expireTime = swapEntry.expireTime();
+
+                    if (expireTime == 0 || expireTime > U.currentTimeMillis()) {
+                        skipEntry = true;
+
+                        ctx.addResult(vals,
+                            cacheKey,
+                            swapEntry.value(),
+                            skipVals,
+                            false,
+                            deserializeBinary,
+                            true,
+                            null,
+                            swapEntry.version(),
+                            0,
+                            0,
+                            needVer);
+
+                        if (configuration().isStatisticsEnabled() && !skipVals)
+                            metrics0().onRead(true);
+
+                        if (evt) {
+                            ctx.events().readEvent(cacheKey,
                                 null,
-                                /*swap*/swapOrOffheap,
-                                /*read-through*/false,
-                                /**update-metrics*/true,
-                                /**event*/!skipVals,
-                                /**temporary*/false,
+                                swapEntry.value(),
                                 subjId,
-                                null,
                                 taskName,
-                                expiry,
                                 !deserializeBinary);
+                        }
+                    }
+                }
+                else
+                    success = false;
+            }
+            else
+                skipEntry = false;
+
+            if (!skipEntry) {
+                GridCacheEntryEx entry = null;
 
-                            if (v != null) {
-                                ctx.addResult(vals,
-                                    cacheKey,
-                                    v,
-                                    skipVals,
-                                    false,
-                                    deserializeBinary,
-                                    true,
+                CacheObject v;
+
+                while (true) {
+                    try {
+                        entry = swapOrOffheap ? entryEx(cacheKey) : peekEx(cacheKey);
+
+                        if (entry != null) {
+                            if (needVer) {
+                                EntryGetResult res = entry.innerGetVersioned(
+                                    null,
                                     null,
-                                    0,
-                                    0);
+                                    /*swap*/swapOrOffheap,
+                                    /*unmarshal*/true,
+                                    /*update-metrics*/false,
+                                    /*event*/!skipVals,
+                                    subjId,
+                                    null,
+                                    taskName,
+                                    expiry,
+                                    !deserializeBinary,
+                                    null);
+
+                                if (res != null) {
+                                    ctx.addResult(
+                                        vals,
+                                        cacheKey,
+                                        res,
+                                        skipVals,
+                                        false,
+                                        deserializeBinary,
+                                        true,
+                                        needVer);
+                                }
+                                else
+                                    success = false;
+                            }
+                            else {
+                                v = entry.innerGet(
+                                    null,
+                                    null,
+                                    /*swap*/swapOrOffheap,
+                                    /*read-through*/false,
+                                    /*update-metrics*/true,
+                                    /*event*/!skipVals,
+                                    /*temporary*/false,
+                                    subjId,
+                                    null,
+                                    taskName,
+                                    expiry,
+                                    !deserializeBinary);
+
+                                if (v != null) {
+                                    ctx.addResult(vals,
+                                        cacheKey,
+                                        v,
+                                        skipVals,
+                                        false,
+                                        deserializeBinary,
+                                        true,
+                                        null,
+                                        0,
+                                        0);
+                                }
+                                else
+                                    success = false;
                             }
-                            else
-                                success = false;
                         }
-                    }
-                    else {
-                        if (!storeEnabled && configuration().isStatisticsEnabled() && !skipVals)
-                            metrics0().onRead(false);
+                        else {
+                            if (!storeEnabled && configuration().isStatisticsEnabled() && !skipVals)
+                                metrics0().onRead(false);
 
-                        success = false;
-                    }
+                            success = false;
+                        }
 
-                    break; // While.
-                }
-                catch (GridCacheEntryRemovedException ignored) {
-                    // No-op, retry.
-                }
-                finally {
-                    if (entry != null)
-                        ctx.evicts().touch(entry, ctx.affinity().affinityTopologyVersion());
+                        break; // While.
+                    }
+                    catch (GridCacheEntryRemovedException ignored) {
+                        // No-op, retry.
+                    }
+                    finally {
+                        if (entry != null)
+                            ctx.evicts().touch(entry, ctx.affinity().affinityTopologyVersion());
+                    }
                 }
 
                 if (!success && storeEnabled)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index bd806aa1d5c17..76979156b0c49 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -2672,17 +2672,17 @@ private boolean enlistWriteEntry(GridCacheContext cacheCtx,
     /**
      * @param cctx Cache context.
      * @param key Key.
-     * @param val Value.
+     * @param val0 Value.
      * @param filter Filter.
      * @return {@code True} if filter passed.
      */
     private boolean isAll(GridCacheContext cctx,
         KeyCacheObject key,
-        CacheObject val,
+        final CacheObject val0,
         CacheEntryPredicate[] filter) {
-        GridCacheEntryEx e = new GridDhtDetachedCacheEntry(cctx, key, 0, val, null, 0) {
+        GridCacheEntryEx e = new GridDhtDetachedCacheEntry(cctx, key, 0) {
             @Nullable @Override public CacheObject peekVisibleValue() {
-                return rawGet();
+                return val0;
             }
         };
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
index 8d1f67af2c33f..eb565b63b5eed 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
@@ -40,11 +40,19 @@ public class CacheRebalancingSelfTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
     /**
      * @throws Exception If failed.
      */
     public void testRebalanceFuture() throws Exception {
         IgniteEx ignite0 = startGrid(0);
+
         startGrid(1);
 
         IgniteCache<Object, Object> cache = ignite0.cache(null);
@@ -63,13 +71,12 @@ public void testRebalanceFuture() throws Exception {
     }
 
     /**
-     * @param future Future.
+     * @param fut Future.
      * @return Internal future.
      */
-    private static IgniteInternalFuture internalFuture(IgniteFuture future) {
-        assert future instanceof IgniteFutureImpl;
+    private static IgniteInternalFuture internalFuture(IgniteFuture fut) {
+        assert fut instanceof IgniteFutureImpl;
 
-        return ((IgniteFutureImpl)future).internalFuture();
+        return ((IgniteFutureImpl) fut).internalFuture();
     }
-
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java
new file mode 100644
index 0000000000000..3e624a3cf0391
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import javax.cache.expiry.Duration;
+import javax.cache.expiry.ModifiedExpiryPolicy;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheEntry;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMemoryMode.OFFHEAP_TIERED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+@SuppressWarnings("unchecked")
+public class IgniteCacheNoSyncForGetTest extends GridCommonAbstractTest {
+    /** IP finder. */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static volatile CountDownLatch processorStartLatch;
+
+    /** */
+    private static volatile CountDownLatch hangLatch;
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid(0);
+
+        client = true;
+
+        startGrid(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        super.afterTestsStopped();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicGetOffheap() throws Exception {
+        boolean getAll[] = {true, false};
+        boolean cfgExpiryPlc[] = {true, false};
+        boolean withExpiryPlc[] = {true, false};
+
+        for (boolean getAll0 : getAll) {
+            for (boolean expiryPlc0 : cfgExpiryPlc)
+                for (boolean withExpiryPlc0 : withExpiryPlc)
+                    doGet(ATOMIC, OFFHEAP_TIERED, getAll0, expiryPlc0, withExpiryPlc0);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxGetOffheap() throws Exception {
+        boolean getAll[] = {true, false};
+        boolean cfgExpiryPlc[] = {true, false};
+        boolean withExpiryPlc[] = {true, false};
+
+        for (boolean getAll0 : getAll) {
+            for (boolean expiryPlc0 : cfgExpiryPlc)
+                for (boolean withExpiryPlc0 : withExpiryPlc)
+                    doGet(TRANSACTIONAL, OFFHEAP_TIERED, getAll0, expiryPlc0, withExpiryPlc0);
+        }
+    }
+
+    /**
+     * @param atomicityMode Cache atomicity mode.
+     * @param memoryMode Cache memory mode.
+     * @param getAll Test getAll flag.
+     * @param cfgExpiryPlc Configured expiry policy flag.
+     * @param withExpiryPlc Custom expiry policy flag.
+     * @throws Exception If failed.
+     */
+    private void doGet(CacheAtomicityMode atomicityMode,
+        CacheMemoryMode memoryMode,
+        final boolean getAll,
+        final boolean cfgExpiryPlc,
+        final boolean withExpiryPlc) throws Exception {
+        log.info("Test get [getAll=" + getAll + ", cfgExpiryPlc=" + cfgExpiryPlc + ']');
+
+        Ignite srv = ignite(0);
+
+        Ignite client = ignite(1);
+
+        final IgniteCache cache = client.createCache(cacheConfiguration(atomicityMode, memoryMode, cfgExpiryPlc));
+
+        final Map<Object, Object> data = new HashMap<>();
+
+        data.put(1, 1);
+        data.put(2, 2);
+
+        try {
+            // Get from compute closure.
+            {
+                cache.putAll(data);
+
+                hangLatch = new CountDownLatch(1);
+                processorStartLatch = new CountDownLatch(1);
+
+                IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
+                    @Override public Void call() throws Exception {
+                        if (getAll)
+                            cache.invokeAll(data.keySet(), new HangEntryProcessor());
+                        else
+                            cache.invoke(1, new HangEntryProcessor());
+
+                        return null;
+                    }
+                });
+
+                try {
+                    boolean wait = processorStartLatch.await(30, TimeUnit.SECONDS);
+
+                    assertTrue(wait);
+
+                    if (getAll) {
+                        assertEquals(data, client.compute().affinityCall(cache.getName(), 1,
+                            new GetAllClosure(data.keySet(), cache.getName(), withExpiryPlc)));
+                    }
+                    else {
+                        assertEquals(1, client.compute().affinityCall(cache.getName(), 1,
+                            new GetClosure(1, cache.getName(), withExpiryPlc)));
+                    }
+
+                    hangLatch.countDown();
+
+                    fut.get();
+                }
+                finally {
+                    hangLatch.countDown();
+                }
+            }
+
+            // Local get.
+            {
+                cache.putAll(data);
+
+                hangLatch = new CountDownLatch(1);
+                processorStartLatch = new CountDownLatch(1);
+
+                IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
+                    @Override public Void call() throws Exception {
+                        if (getAll)
+                            cache.invokeAll(data.keySet(), new HangEntryProcessor());
+                        else
+                            cache.invoke(1, new HangEntryProcessor());
+
+                        return null;
+                    }
+                });
+
+                try {
+                    boolean wait = processorStartLatch.await(30, TimeUnit.SECONDS);
+
+                    assertTrue(wait);
+
+                    IgniteCache srvCache = srv.cache(cache.getName());
+
+                    if (withExpiryPlc)
+                        srvCache = srvCache.withExpiryPolicy(ModifiedExpiryPolicy.factoryOf(Duration.FIVE_MINUTES).create());
+
+                    if (getAll) {
+                        assertEquals(data, srvCache.getAll(data.keySet()));
+                        assertEquals(data.size(), srvCache.getEntries(data.keySet()).size());
+                    }
+                    else {
+                        assertEquals(1, srvCache.get(1));
+                        assertEquals(1, srvCache.getEntry(1).getValue());
+                    }
+
+                    hangLatch.countDown();
+
+                    fut.get();
+                }
+                finally {
+                    hangLatch.countDown();
+                }
+            }
+        }
+        finally {
+            client.destroyCache(cache.getName());
+        }
+    }
+
+    /**
+     * @param atomicityMode Atomicity mode.
+     * @param memoryMode Memory mode.
+     * @param expiryPlc Expiry policy flag.
+     * @return Cache configuration.
+     */
+    private CacheConfiguration cacheConfiguration(CacheAtomicityMode atomicityMode,
+        CacheMemoryMode memoryMode,
+        boolean expiryPlc) {
+        CacheConfiguration ccfg = new CacheConfiguration();
+
+        ccfg.setAtomicityMode(atomicityMode);
+        ccfg.setMemoryMode(memoryMode);
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+        ccfg.setName("testCache");
+
+        if (expiryPlc)
+            ccfg.setExpiryPolicyFactory(ModifiedExpiryPolicy.factoryOf(Duration.FIVE_MINUTES));
+
+        return ccfg;
+    }
+
+    /**
+     *
+     */
+    static class HangEntryProcessor implements CacheEntryProcessor {
+        /** {@inheritDoc} */
+        @Override public Object process(MutableEntry entry, Object... arguments) {
+            assert processorStartLatch != null;
+            assert hangLatch != null;
+
+            try {
+                processorStartLatch.countDown();
+
+                if (!hangLatch.await(60, TimeUnit.SECONDS))
+                    throw new RuntimeException("Failed to wait for latch");
+            }
+            catch (Exception e) {
+                System.out.println("Unexpected error: " + e);
+
+                throw new EntryProcessorException(e);
+            }
+
+            entry.setValue(U.currentTimeMillis());
+
+            return null;
+        }
+    }
+
+    /**
+     *
+     */
+    public static class GetClosure implements IgniteCallable<Object> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** */
+        private final int key;
+
+        /** */
+        private final String cacheName;
+
+        /** */
+        private final boolean withExpiryPlc;
+
+        /**
+         * @param key Key.
+         * @param cacheName Cache name.
+         * @param withExpiryPlc Custom expiry policy flag.
+         */
+        GetClosure(int key, String cacheName, boolean withExpiryPlc) {
+            this.key = key;
+            this.cacheName = cacheName;
+            this.withExpiryPlc = withExpiryPlc;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object call() throws Exception {
+            IgniteCache cache = ignite.cache(cacheName);
+
+            if (withExpiryPlc)
+                cache = cache.withExpiryPolicy(ModifiedExpiryPolicy.factoryOf(Duration.FIVE_MINUTES).create());
+
+            Object val = cache.get(key);
+
+            CacheEntry e = cache.getEntry(key);
+
+            assertEquals(val, e.getValue());
+
+            return val;
+        }
+    }
+
+    /**
+     *
+     */
+    public static class GetAllClosure implements IgniteCallable<Object> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** */
+        private final Set<Object> keys;
+
+        /** */
+        private final String cacheName;
+
+        /** */
+        private final boolean withExpiryPlc;
+
+        /**
+         * @param keys Keys.
+         * @param cacheName Cache name.
+         * @param withExpiryPlc Custom expiry policy flag.
+         */
+        GetAllClosure(Set<Object> keys, String cacheName, boolean withExpiryPlc) {
+            this.keys = keys;
+            this.cacheName = cacheName;
+            this.withExpiryPlc = withExpiryPlc;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object call() throws Exception {
+            IgniteCache cache = ignite.cache(cacheName);
+
+            if (withExpiryPlc)
+                cache = cache.withExpiryPolicy(ModifiedExpiryPolicy.factoryOf(Duration.FIVE_MINUTES).create());
+
+            Map vals = cache.getAll(keys);
+
+            Collection<CacheEntry> entries = cache.getEntries(keys);
+
+            assertEquals(vals.size(), entries.size());
+
+            for (CacheEntry entry : entries) {
+                Object val = vals.get(entry.getKey());
+
+                assertEquals(val, entry.getValue());
+            }
+
+            return vals;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index 8f08ea95bad7b..423c1ba9a6d39 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -1026,7 +1026,7 @@ public void testNearExpiresOnClient() throws Exception {
         if(cacheMode() != PARTITIONED)
             return;
 
-        factory =  CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS,1));
+        factory =  CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.SECONDS, 2));
 
         nearCache = true;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridHashMapLoadTest.java b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridHashMapLoadTest.java
index 5c12f8496ee99..0e8d66bc1034f 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridHashMapLoadTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridHashMapLoadTest.java
@@ -79,10 +79,9 @@ public void testMapEntry() throws Exception {
 
         while (true) {
             Integer key = i++;
-            Integer val = i++;
 
             map.put(key, new GridCacheMapEntry(ctx, ctx.toCacheKeyObject(key),
-                key.hashCode(), ctx.toCacheObject(val)) {
+                key.hashCode()) {
                 @Override public boolean tmLock(IgniteInternalTx tx,
                     long timeout,
                     @Nullable GridCacheVersion serOrder,
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
index e37a8a1d52096..6d0745dea8a74 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteAtomicCacheEntryProcessorNodeJoinTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheEntryProcessorNodeJoinTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheIncrementTxTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheNoSyncForGetTest;
 import org.apache.ignite.internal.processors.cache.IgniteCachePartitionMapUpdateTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheAndNodeStop;
 import org.apache.ignite.internal.processors.cache.OffheapCacheOnClientsTest;
@@ -114,11 +115,11 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxSingleThreadedSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheRendezvousAffinityClientSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridPartitionedBackupLoadSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheStoreUpdateTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearOffheapCacheStoreUpdateTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridPartitionedBackupLoadSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.NearCacheSyncUpdateTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.NoneRebalanceModeSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridNearOffheapCacheStoreUpdateTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedEvictionSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedJobExecutionTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicBasicStoreSelfTest;
@@ -278,6 +279,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(GridNearCacheStoreUpdateTest.class));
         suite.addTest(new TestSuite(GridNearOffheapCacheStoreUpdateTest.class));
 
+        suite.addTest(new TestSuite(IgniteCacheNoSyncForGetTest.class));
+
         return suite;
     }
 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlAbstractSelfTest.java
index ace7d94ee7854..402230d2e5414 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlAbstractSelfTest.java
@@ -69,7 +69,7 @@ public abstract class CacheTtlAbstractSelfTest extends GridCommonAbstractTest {
     private static final int SIZE = 11;
 
     /** */
-    private static final long DEFAULT_TIME_TO_LIVE = 2000;
+    private static final long DEFAULT_TIME_TO_LIVE = 4000;
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {

From 320def0b0cc357cfae4423352623ead7aa1d5984 Mon Sep 17 00:00:00 2001
From: Alexander Belyak <alexandr.belyak@xored.com>
Date: Tue, 18 Apr 2017 14:56:50 +0300
Subject: [PATCH 132/516] IGNITE-4927: Write behind - add an option to skip
 write coalescing

---
 .../configuration/CacheConfiguration.java     |  33 +
 .../store/GridCacheWriteBehindStore.java      | 614 +++++++++++++++---
 ...CacheWriteBehindStoreAbstractSelfTest.java |  24 +-
 ...GridCacheWriteBehindStoreAbstractTest.java |   4 +
 ...WriteBehindStoreMultithreadedSelfTest.java |  88 ++-
 .../GridCacheWriteBehindStoreSelfTest.java    | 159 ++++-
 ...ientWriteBehindStoreNonCoalescingTest.java | 175 +++++
 .../IgniteCacheWriteBehindTestSuite.java      |   2 +
 8 files changed, 978 insertions(+), 121 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
index e160ba9963e22..31806e73e69e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
@@ -192,6 +192,9 @@ public class CacheConfiguration<K, V> extends MutableConfiguration<K, V> {
     /** Default batch size for write-behind cache store. */
     public static final int DFLT_WRITE_BEHIND_BATCH_SIZE = 512;
 
+    /** Default write coalescing for write-behind cache store. */
+    public static final boolean DFLT_WRITE_BEHIND_COALESCING = true;
+
     /** Default maximum number of query iterators that can be stored. */
     public static final int DFLT_MAX_QUERY_ITERATOR_CNT = 1024;
 
@@ -344,6 +347,9 @@ public class CacheConfiguration<K, V> extends MutableConfiguration<K, V> {
     /** Maximum batch size for write-behind cache store. */
     private int writeBehindBatchSize = DFLT_WRITE_BEHIND_BATCH_SIZE;
 
+    /** Write coalescing flag for write-behind cache store */
+    private boolean writeBehindCoalescing = DFLT_WRITE_BEHIND_COALESCING;
+
     /** Maximum number of query iterators that can be stored. */
     private int maxQryIterCnt = DFLT_MAX_QUERY_ITERATOR_CNT;
 
@@ -505,6 +511,7 @@ public CacheConfiguration(CompleteConfiguration<K, V> cfg) {
         topValidator = cc.getTopologyValidator();
         typeMeta = cc.getTypeMetadata();
         writeBehindBatchSize = cc.getWriteBehindBatchSize();
+        writeBehindCoalescing = cc.getWriteBehindCoalescing();
         writeBehindEnabled = cc.isWriteBehindEnabled();
         writeBehindFlushFreq = cc.getWriteBehindFlushFrequency();
         writeBehindFlushSize = cc.getWriteBehindFlushSize();
@@ -1460,6 +1467,32 @@ public CacheConfiguration<K, V> setWriteBehindBatchSize(int writeBehindBatchSize
         return this;
     }
 
+    /**
+     * Write coalescing flag for write-behind cache store operations. Store operations (get or remove)
+     * with the same key are combined or coalesced to single, resulting operation
+     * to reduce pressure to underlying cache store.
+     * <p/>
+     * If not provided, default value is {@link #DFLT_WRITE_BEHIND_COALESCING}.
+     *
+     * @return Write coalescing flag.
+     */
+    public boolean getWriteBehindCoalescing() {
+        return writeBehindCoalescing;
+    }
+
+    /**
+     * Sets write coalescing flag for write-behind cache.
+     *
+     * @param writeBehindCoalescing Write coalescing flag.
+     * @see #getWriteBehindCoalescing()
+     * @return {@code this} for chaining.
+     */
+    public CacheConfiguration<K, V> setWriteBehindCoalescing(boolean writeBehindCoalescing) {
+        this.writeBehindCoalescing = writeBehindCoalescing;
+
+        return this;
+    }
+
     /**
      * Use {@link IgniteConfiguration#getRebalanceThreadPoolSize()} instead.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
index 0f70da5e7e160..4a1b11e807bf3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
@@ -17,17 +17,19 @@
 
 package org.apache.ignite.internal.processors.cache.store;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.LockSupport;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import javax.cache.integration.CacheWriterException;
@@ -43,9 +45,11 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.lang.IgniteBiInClosure;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lifecycle.LifecycleAware;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
+import org.jsr166.ConcurrentLinkedDeque8;
 import org.jsr166.ConcurrentLinkedHashMap;
 
 import static javax.cache.Cache.Entry;
@@ -65,6 +69,8 @@
  * <p/>
  * Since write operations to the cache store are deferred, transaction support is lost; no
  * transaction objects are passed to the underlying store.
+ * <p/>
+ * {@link GridCacheWriteBehindStore} doesn't support concurrent modifications of the same key.
  */
 public class GridCacheWriteBehindStore<K, V> implements CacheStore<K, V>, LifecycleAware {
     /** Default write cache initial capacity. */
@@ -91,6 +97,9 @@ public class GridCacheWriteBehindStore<K, V> implements CacheStore<K, V>, Lifecy
     /** Count of worker threads performing underlying store updates. */
     private int flushThreadCnt = CacheConfiguration.DFLT_WRITE_FROM_BEHIND_FLUSH_THREAD_CNT;
 
+    /** Is flush threads count power of two flag. */
+    private boolean flushThreadCntIsPowerOfTwo;
+
     /** Cache flush frequency. All pending operations will be performed in not less then this value ms. */
     private long cacheFlushFreq = CacheConfiguration.DFLT_WRITE_BEHIND_FLUSH_FREQUENCY;
 
@@ -98,29 +107,26 @@ public class GridCacheWriteBehindStore<K, V> implements CacheStore<K, V>, Lifecy
     private int batchSize = CacheConfiguration.DFLT_WRITE_BEHIND_BATCH_SIZE;
 
     /** Grid name. */
-    private String gridName;
+    private final String gridName;
 
     /** Cache name. */
-    private String cacheName;
+    private final String cacheName;
 
     /** Underlying store. */
-    private CacheStore<K, V> store;
+    private final CacheStore<K, V> store;
 
     /** Write cache. */
     private ConcurrentLinkedHashMap<K, StatefulValue<K, V>> writeCache;
 
     /** Flusher threads. */
-    private GridWorker[] flushThreads;
+    private Flusher[] flushThreads;
+
+    /** Write coalescing. */
+    private boolean writeCoalescing = CacheConfiguration.DFLT_WRITE_BEHIND_COALESCING;
 
     /** Atomic flag indicating store shutdown. */
     private AtomicBoolean stopping = new AtomicBoolean(true);
 
-    /** Flush lock. */
-    private Lock flushLock = new ReentrantLock();
-
-    /** Condition to determine records available for flush. */
-    private Condition canFlush = flushLock.newCondition();
-
     /** Variable for counting total cache overflows. */
     private AtomicInteger cacheTotalOverflowCntr = new AtomicInteger();
 
@@ -131,10 +137,16 @@ public class GridCacheWriteBehindStore<K, V> implements CacheStore<K, V>, Lifecy
     private AtomicInteger retryEntriesCnt = new AtomicInteger();
 
     /** Log. */
-    private IgniteLogger log;
+    private final IgniteLogger log;
 
     /** Store manager. */
-    private CacheStoreManager storeMgr;
+    private final CacheStoreManager storeMgr;
+
+    /** Flush lock. */
+    private final Lock flushLock = new ReentrantLock();
+
+    /** Condition to determine records available for flush. */
+    private Condition canFlush = flushLock.newCondition();
 
     /**
      * Creates a write-behind cache store for the given store.
@@ -193,7 +205,7 @@ public void setFlushSize(int cacheMaxSize) {
      * <p/>
      * If this value is {@code 0}, then flush is performed only on time-elapsing basis. However,
      * when this value is {@code 0}, the cache critical size is set to
-     * {@link CacheConfiguration#DFLT_WRITE_BEHIND_CRITICAL_SIZE}
+     * {@link CacheConfiguration#DFLT_WRITE_BEHIND_CRITICAL_SIZE}.
      *
      * @return Buffer size that triggers flush procedure.
      */
@@ -208,6 +220,7 @@ public int getWriteBehindFlushSize() {
      */
     public void setFlushThreadCount(int flushThreadCnt) {
         this.flushThreadCnt = flushThreadCnt;
+        this.flushThreadCntIsPowerOfTwo = (flushThreadCnt & (flushThreadCnt - 1)) == 0;
     }
 
     /**
@@ -219,6 +232,24 @@ public int getWriteBehindFlushThreadCount() {
         return flushThreadCnt;
     }
 
+    /**
+     * Sets the write coalescing flag.
+     *
+     * @param writeCoalescing Write coalescing flag.
+     */
+    public void setWriteCoalescing(boolean writeCoalescing) {
+        this.writeCoalescing = writeCoalescing;
+    }
+
+    /**
+     * Gets the write coalescing flag.
+     *
+     * @return Write coalescing flag.
+     */
+    public boolean getWriteCoalescing() {
+        return writeCoalescing;
+    }
+
     /**
      * Sets the cache flush frequency. All pending operations on the underlying store will be performed
      * within time interval not less then this value.
@@ -266,7 +297,16 @@ public int getWriteBehindStoreBatchSize() {
      * @return Total count of entries in cache store internal buffer.
      */
     public int getWriteBehindBufferSize() {
-        return writeCache.sizex();
+        if (writeCoalescing)
+            return writeCache.sizex();
+        else {
+            int size = 0;
+
+            for (Flusher f : flushThreads)
+                size += f.size();
+
+            return size;
+        }
     }
 
     /**
@@ -292,14 +332,15 @@ public CacheStore<K, V> store() {
             if (cacheCriticalSize == 0)
                 cacheCriticalSize = CacheConfiguration.DFLT_WRITE_BEHIND_CRITICAL_SIZE;
 
-            flushThreads = new GridWorker[flushThreadCnt];
+            flushThreads = new GridCacheWriteBehindStore.Flusher[flushThreadCnt];
 
-            writeCache = new ConcurrentLinkedHashMap<>(initCap, 0.75f, concurLvl);
+            if (writeCoalescing)
+                writeCache = new ConcurrentLinkedHashMap<>(initCap, 0.75f, concurLvl);
 
             for (int i = 0; i < flushThreads.length; i++) {
                 flushThreads[i] = new Flusher(gridName, "flusher-" + i, log);
 
-                new IgniteThread(flushThreads[i]).start();
+                flushThreads[i].start();
             }
         }
     }
@@ -344,7 +385,10 @@ public int getWriteBehindErrorRetryCount() {
             if (log.isDebugEnabled())
                 log.debug("Stopping write-behind store for cache '" + cacheName + '\'');
 
-            wakeUp();
+            for (Flusher f : flushThreads) {
+                if (!f.isEmpty())
+                    f.wakeUp();
+            }
 
             boolean graceful = true;
 
@@ -352,7 +396,7 @@ public int getWriteBehindErrorRetryCount() {
                 graceful &= U.join(worker, log);
 
             if (!graceful)
-                log.warning("Shutdown was aborted");
+                log.warning("Write behind store shutdown was aborted.");
         }
     }
 
@@ -361,7 +405,10 @@ public int getWriteBehindErrorRetryCount() {
      * @throws IgniteCheckedException If failed.
      */
     public void forceFlush() throws IgniteCheckedException {
-        wakeUp();
+        for (Flusher f : flushThreads) {
+            if (!f.isEmpty())
+                f.wakeUp();
+        }
     }
 
     /** {@inheritDoc} */
@@ -376,10 +423,15 @@ public void forceFlush() throws IgniteCheckedException {
 
         Map<K, V> loaded = new HashMap<>();
 
-        Collection<K> remaining = new LinkedList<>();
+        Collection<K> remaining = null;
 
         for (K key : keys) {
-            StatefulValue<K, V> val = writeCache.get(key);
+            StatefulValue<K, V> val;
+
+            if (writeCoalescing)
+                val = writeCache.get(key);
+            else
+                val = flusher(key).flusherWriteMap.get(key);
 
             if (val != null) {
                 val.readLock().lock();
@@ -394,12 +446,16 @@ public void forceFlush() throws IgniteCheckedException {
                     val.readLock().unlock();
                 }
             }
-            else
+            else {
+                if (remaining == null)
+                    remaining = new ArrayList<>();
+
                 remaining.add(key);
+            }
         }
 
         // For items that were not found in queue.
-        if (!remaining.isEmpty()) {
+        if (remaining != null && !remaining.isEmpty()) {
             Map<K, V> loaded0 = store.loadAll(remaining);
 
             if (loaded0 != null)
@@ -414,7 +470,12 @@ public void forceFlush() throws IgniteCheckedException {
         if (log.isDebugEnabled())
             log.debug("Store load [key=" + key + ']');
 
-        StatefulValue<K, V> val = writeCache.get(key);
+        StatefulValue<K, V> val;
+
+        if (writeCoalescing)
+            val = writeCache.get(key);
+        else
+            val = flusher(key).flusherWriteMap.get(key);
 
         if (val != null) {
             val.readLock().lock();
@@ -493,7 +554,7 @@ public void forceFlush() throws IgniteCheckedException {
      *
      * @param key Key for which update is performed.
      * @param val New value, may be null for remove operation.
-     * @param operation Updated value status
+     * @param operation Updated value status.
      * @throws IgniteInterruptedCheckedException If interrupted while waiting for value to be flushed.
      */
     private void updateCache(K key,
@@ -502,8 +563,27 @@ private void updateCache(K key,
         throws IgniteInterruptedCheckedException {
         StatefulValue<K, V> newVal = new StatefulValue<>(val, operation);
 
+        if (writeCoalescing)
+            putToWriteCache(key, newVal);
+        else
+            flusher(key).putToFlusherWriteCache(key, newVal);
+    }
+
+    /**
+     * Performs flush-consistent writeCache update for the given key.
+     *
+     * @param key Key for which update is performed.
+     * @param newVal stateful value to put
+     * @throws IgniteInterruptedCheckedException If interrupted while waiting for value to be flushed.
+     */
+    private void putToWriteCache(
+        K key,
+        StatefulValue<K, V> newVal)
+        throws IgniteInterruptedCheckedException {
         StatefulValue<K, V> prev;
 
+        assert writeCoalescing : "Unexpected write coalescing.";
+
         while ((prev = writeCache.putIfAbsent(key, newVal)) != null) {
             prev.writeLock().lock();
 
@@ -523,7 +603,7 @@ else if (prev.status() == ValueStatus.RETRY)
 
                 assert prev.status() == ValueStatus.NEW || prev.status() == ValueStatus.RETRY;
 
-                prev.update(val, operation, ValueStatus.NEW);
+                prev.update(newVal.val, newVal.operation(), ValueStatus.NEW);
 
                 break;
             }
@@ -533,13 +613,32 @@ else if (prev.status() == ValueStatus.RETRY)
         }
 
         // Now check the map size
-        if (writeCache.sizex() > cacheCriticalSize)
+        int cacheSize = getWriteBehindBufferSize();
+
+        if (cacheSize > cacheCriticalSize)
             // Perform single store update in the same thread.
             flushSingleValue();
-        else if (cacheMaxSize > 0 && writeCache.sizex() > cacheMaxSize)
+        else if (cacheMaxSize > 0 && cacheSize > cacheMaxSize)
             wakeUp();
     }
 
+    /**
+     * Return flusher by by key.
+     *
+     * @param key Key for search.
+     * @return flusher.
+     */
+    private Flusher flusher(K key) {
+        int h, idx;
+
+        if (flushThreadCntIsPowerOfTwo)
+            idx = ((h = key.hashCode()) ^ (h >>> 16)) & (flushThreadCnt - 1);
+        else
+            idx = ((h = key.hashCode()) ^ (h >>> 16)) % flushThreadCnt;
+
+        return flushThreads[idx];
+    }
+
     /**
      * Flushes one upcoming value to the underlying store. Called from
      * {@link #updateCache(Object, Entry, StoreOperation)} method in case when current map size exceeds
@@ -549,7 +648,7 @@ private void flushSingleValue() {
         cacheOverflowCntr.incrementAndGet();
 
         try {
-            Map<K, StatefulValue<K, V>> batch = null;
+            Map<K, StatefulValue<K, V>> batch;
 
             for (Map.Entry<K, StatefulValue<K, V>> e : writeCache.entrySet()) {
                 StatefulValue<K, V> val = e.getValue();
@@ -577,7 +676,7 @@ private void flushSingleValue() {
                 }
 
                 if (!batch.isEmpty()) {
-                    applyBatch(batch, false);
+                    applyBatch(batch, false, null);
 
                     cacheTotalOverflowCntr.incrementAndGet();
 
@@ -595,9 +694,12 @@ private void flushSingleValue() {
      *
      * @param valMap Batch map.
      * @param initSes {@code True} if need to initialize session.
+     * @param flusher Flusher, assotiated with all keys in batch (have sense in write coalescing = false mode)
+     * @return {@code True} if batch was successfully applied, {@code False} otherwise.
      */
-    private void applyBatch(Map<K, StatefulValue<K, V>> valMap, boolean initSes) {
+    private boolean applyBatch(Map<K, StatefulValue<K, V>> valMap, boolean initSes, Flusher flusher) {
         assert valMap.size() <= batchSize;
+        assert !valMap.isEmpty();
 
         StoreOperation operation = null;
 
@@ -615,7 +717,9 @@ private void applyBatch(Map<K, StatefulValue<K, V>> valMap, boolean initSes) {
             batch.put(e.getKey(), e.getValue().entry());
         }
 
-        if (updateStore(operation, batch, initSes)) {
+        boolean result = updateStore(operation, batch, initSes, flusher);
+
+        if (result) {
             for (Map.Entry<K, StatefulValue<K, V>> e : valMap.entrySet()) {
                 StatefulValue<K, V> val = e.getValue();
 
@@ -624,12 +728,22 @@ private void applyBatch(Map<K, StatefulValue<K, V>> valMap, boolean initSes) {
                 try {
                     val.status(ValueStatus.FLUSHED);
 
-                    StatefulValue<K, V> prev = writeCache.remove(e.getKey());
+                    if (writeCoalescing) {
+                        StatefulValue<K, V> prev = writeCache.remove(e.getKey());
 
-                    // Additional check to ensure consistency.
-                    assert prev == val : "Map value for key " + e.getKey() + " was updated during flush";
+                        // Additional check to ensure consistency.
+                        assert prev == val : "Map value for key " + e.getKey() + " was updated during flush";
 
-                    val.signalFlushed();
+                        val.signalFlushed();
+                    }
+                    else {
+                        Flusher f = flusher(e.getKey());
+
+                        // Can remove using equal because if map contains another similar value it has different state.
+                        f.flusherWriteMap.remove(e.getKey(), e.getValue());
+
+                        val.signalFlushed();
+                    }
                 }
                 finally {
                     val.writeLock().unlock();
@@ -653,6 +767,8 @@ private void applyBatch(Map<K, StatefulValue<K, V>> valMap, boolean initSes) {
                 }
             }
         }
+
+        return result;
     }
 
     /**
@@ -666,13 +782,16 @@ private void applyBatch(Map<K, StatefulValue<K, V>> valMap, boolean initSes) {
      * @param operation Status indicating operation that should be performed.
      * @param vals Key-Value map.
      * @param initSes {@code True} if need to initialize session.
+     * @param flusher Flusher, assotiated with vals keys (in writeCoalescing=false mode)
      * @return {@code true} if value may be deleted from the write cache,
      *         {@code false} otherwise
      */
-    private boolean updateStore(StoreOperation operation,
+    private boolean updateStore(
+        StoreOperation operation,
         Map<K, Entry<? extends K, ? extends  V>> vals,
-        boolean initSes) {
-
+        boolean initSes,
+        Flusher flusher
+    ) {
         try {
             if (initSes && storeMgr != null)
                 storeMgr.writeBehindSessionInit();
@@ -707,7 +826,14 @@ private boolean updateStore(StoreOperation operation,
         catch (Exception e) {
             LT.error(log, e, "Unable to update underlying store: " + store);
 
-            if (writeCache.sizex() > cacheCriticalSize || stopping.get()) {
+            boolean overflow;
+
+            if (writeCoalescing)
+                overflow = writeCache.sizex() > cacheCriticalSize || stopping.get();
+            else
+                overflow = flusher.isOverflowed() || stopping.get();
+
+            if (overflow) {
                 for (Map.Entry<K, Entry<? extends K, ? extends  V>> entry : vals.entrySet()) {
                     Object val = entry.getValue() != null ? entry.getValue().getValue() : null;
 
@@ -738,29 +864,163 @@ private void wakeUp() {
     }
 
     /**
-     * Thread that performs time-based flushing of written values to the underlying storage.
+     * Thread that performs time/size-based flushing of written values to the underlying storage.
      */
     private class Flusher extends GridWorker {
+        /** Queue to flush. */
+        private final ConcurrentLinkedDeque8<IgniteBiTuple<K, StatefulValue<K,V>>> queue;
+
+        /** Flusher write map. */
+        private final ConcurrentHashMap<K, StatefulValue<K,V>> flusherWriteMap;
+
+        /** Critical size of flusher local queue. */
+        private final int flusherCacheCriticalSize;
+
+        /** Flusher parked flag. */
+        private volatile boolean parked;
+
+        /** Flusher thread. */
+        protected Thread thread;
+
+        /** Cache flushing frequence in nanos. */
+        protected long cacheFlushFreqNanos = cacheFlushFreq * 1000;
+
+        /** Writer lock. */
+        private final Lock flusherWriterLock = new ReentrantLock();
+
+        /** Confition to determine available space for flush. */
+        private Condition flusherWriterCanWrite = flusherWriterLock.newCondition();
+
         /** {@inheritDoc */
-        protected Flusher(String gridName, String name, IgniteLogger log) {
+        protected Flusher(String gridName,
+            String name,
+            IgniteLogger log) {
             super(gridName, name, log);
+
+            flusherCacheCriticalSize = cacheCriticalSize/flushThreadCnt;
+
+            assert flusherCacheCriticalSize > batchSize;
+
+            if (writeCoalescing) {
+                queue = null;
+                flusherWriteMap = null;
+            }
+            else {
+                queue = new ConcurrentLinkedDeque8<>();
+                flusherWriteMap = new ConcurrentHashMap<>(initCap, 0.75f, concurLvl);
+            }
+        }
+
+        /** Start flusher thread */
+        protected void start() {
+            thread = new IgniteThread(this);
+            thread.start();
+        }
+
+        /**
+         * Performs flush-consistent flusher writeCache update for the given key.
+         *
+         * @param key Key for which update is performed.
+         * @param newVal stateful value to put
+         * @throws IgniteInterruptedCheckedException If interrupted while waiting for value to be flushed.
+         */
+        private void putToFlusherWriteCache(
+            K key,
+            StatefulValue<K, V> newVal)
+            throws IgniteInterruptedCheckedException {
+            assert !writeCoalescing : "Unexpected write coalescing.";
+
+            if (queue.sizex() > flusherCacheCriticalSize) {
+                while (queue.sizex() > flusherCacheCriticalSize) {
+                    wakeUp();
+
+                    flusherWriterLock.lock();
+
+                    try {
+                        // Wait for free space in flusher queue
+                        while (queue.sizex() >= flusherCacheCriticalSize && !stopping.get()) {
+                            if (cacheFlushFreq > 0)
+                                flusherWriterCanWrite.await(cacheFlushFreq, TimeUnit.MILLISECONDS);
+                            else
+                                flusherWriterCanWrite.await();
+                        }
+
+                        cacheTotalOverflowCntr.incrementAndGet();
+                    }
+                    catch (InterruptedException e) {
+                        if (log.isDebugEnabled())
+                            log.debug("Caught interrupted exception: " + e);
+
+                        Thread.currentThread().interrupt();
+                    }
+                    finally {
+                        flusherWriterLock.unlock();
+                    }
+                }
+
+                cacheTotalOverflowCntr.incrementAndGet();
+            }
+
+            queue.add(F.t(key, newVal));
+
+            flusherWriteMap.put(key, newVal);
+        }
+
+        /**
+         * Get overflowed flag.
+         *
+         * @return {@code True} if write behind flusher is overflowed,
+         *         {@code False} otherwise.
+         */
+        public boolean isOverflowed() {
+            if (writeCoalescing)
+                return writeCache.sizex() > cacheCriticalSize;
+            else
+                return queue.sizex() > flusherCacheCriticalSize;
+        }
+
+        /**
+         * Get write behind flusher size.
+         *
+         * @return Flusher write behind size.
+         */
+        public int size() {
+            return writeCoalescing ? writeCache.sizex() : queue.sizex();
+        }
+
+        /**
+         * Test if write behind flusher is empty
+         *
+         * @return {@code True} if write behind flusher is empty, {@code False} otherwise
+         */
+        public boolean isEmpty() {
+            return writeCoalescing ? writeCache.isEmpty() : queue.isEmpty();
         }
 
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
-            while (!stopping.get() || writeCache.sizex() > 0) {
-                awaitOperationsAvailable();
+            if (writeCoalescing) {
+                while (!stopping.get() || writeCache.sizex() > 0) {
+                    awaitOperationsAvailableCoalescing();
 
-                flushCache(writeCache.entrySet().iterator());
+                    flushCacheCoalescing();
+                }
+            }
+            else {
+                while (!stopping.get() || queue.sizex() > 0) {
+                    awaitOperationsAvailableNonCoalescing();
+
+                    flushCacheNonCoalescing();
+                }
             }
         }
 
         /**
-         * This method awaits until enough elements in map are available or given timeout is over.
+         * This method awaits until enough elements in flusher queue are available or given timeout is over.
          *
          * @throws InterruptedException If awaiting was interrupted.
          */
-        private void awaitOperationsAvailable() throws InterruptedException {
+        private void awaitOperationsAvailableCoalescing() throws InterruptedException {
             flushLock.lock();
 
             try {
@@ -779,75 +1039,216 @@ private void awaitOperationsAvailable() throws InterruptedException {
             }
         }
 
+        /**
+         * This method awaits until enough elements in flusher queue are available or given timeout is over.
+         *
+         * @throws InterruptedException If awaiting was interrupted.
+         */
+        private void awaitOperationsAvailableNonCoalescing() throws InterruptedException {
+            if (queue.sizex() >= batchSize)
+                return;
+
+            parked = true;
+
+            try {
+                for (;;) {
+                    if (queue.sizex() >= batchSize)
+                        return;
+
+                    if (cacheFlushFreq > 0)
+                        LockSupport.parkNanos(cacheFlushFreqNanos);
+                    else
+                        LockSupport.park();
+
+                    if (queue.sizex() > 0)
+                        return;
+
+                    if (Thread.interrupted())
+                        throw new InterruptedException();
+
+                    if (stopping.get())
+                        return;
+                }
+            }
+            finally {
+                parked = false;
+            }
+        }
+
+        /**
+         * Wake up flusher thread.
+         */
+        public void wakeUp() {
+            if (parked)
+                LockSupport.unpark(thread);
+        }
+
         /**
          * Removes values from the write cache and performs corresponding operation
          * on the underlying store.
-         *
-         * @param it Iterator for write cache.
          */
-        private void flushCache(Iterator<Map.Entry<K,StatefulValue<K, V>>> it) {
-            StoreOperation operation = null;
+        private void flushCacheCoalescing() {
+            StoreOperation prevOperation = null;
 
-            Map<K, StatefulValue<K, V>> batch = null;
-            Map<K, StatefulValue<K, V>> pending  = U.newLinkedHashMap(batchSize);
+            Map<K, StatefulValue<K, V>> pending = U.newLinkedHashMap(batchSize);
+            Iterator<Map.Entry<K, StatefulValue<K, V>>> it = writeCache.entrySet().iterator();
 
             while (it.hasNext()) {
                 Map.Entry<K, StatefulValue<K, V>> e = it.next();
-
                 StatefulValue<K, V> val = e.getValue();
 
-                val.writeLock().lock();
+                if (!val.writeLock().tryLock()) // TODO: stripe write maps to avoid lock contention.
+                    continue;
 
                 try {
-                    ValueStatus status = val.status();
+                    BatchingResult addRes = tryAddStatefulValue(pending, prevOperation, e.getKey(), val);
 
-                    if (acquired(status))
-                        // Another thread is helping us, continue to the next entry.
-                        continue;
-
-                    if (status == ValueStatus.RETRY)
-                        retryEntriesCnt.decrementAndGet();
+                    switch (addRes) {
+                        case NEW_BATCH:
+                            applyBatch(pending, true, null);
 
-                    assert retryEntriesCnt.get() >= 0;
+                            pending = U.newLinkedHashMap(batchSize);
 
-                    val.status(ValueStatus.PENDING);
+                            // No need to test first value in batch
+                            val.status(ValueStatus.PENDING);
+                            pending.put(e.getKey(), val);
+                            prevOperation = val.operation();
 
-                    // We scan for the next operation and apply batch on operation change. Null means new batch.
-                    if (operation == null)
-                        operation = val.operation();
+                            break;
 
-                    if (operation != val.operation()) {
-                        // Operation is changed, so we need to perform a batch.
-                        batch = pending;
-                        pending = U.newLinkedHashMap(batchSize);
+                        case ADDED:
+                            prevOperation = val.operation();
 
-                        operation = val.operation();
+                            break;
 
-                        pending.put(e.getKey(), val);
+                        default:
+                            assert addRes == BatchingResult.SKIPPED : "Unexpected result: " + addRes;
                     }
-                    else
-                        pending.put(e.getKey(), val);
+                }
+                finally {
+                    val.writeLock().unlock();
+                }
+            }
+
+            // Process the remainder.
+            if (!pending.isEmpty())
+                applyBatch(pending, true, null);
+        }
+
+        /**
+         * Removes values from the flusher write queue and performs corresponding operation
+         * on the underlying store.
+         */
+        private void flushCacheNonCoalescing() {
+            StoreOperation prevOperation;
+            Map<K, StatefulValue<K, V>> pending;
+            IgniteBiTuple<K, StatefulValue<K, V>> tuple;
+            boolean applied;
+
+            while(!queue.isEmpty()) {
+                pending = U.newLinkedHashMap(batchSize);
+                prevOperation = null;
+                boolean needNewBatch = false;
+
+                // Collect batch
+                while (!needNewBatch && (tuple = queue.peek()) != null) {
+                    BatchingResult addRes = tryAddStatefulValue(pending, prevOperation, tuple.getKey(),
+                        tuple.getValue());
+
+                    switch (addRes) {
+                        case ADDED:
+                            prevOperation = tuple.getValue().operation();
+                            queue.poll();
+
+                            break;
+
+                        case SKIPPED:
+                            assert false : "Unexpected result: " + addRes;
+
+                            break;
 
-                    if (pending.size() == batchSize) {
-                        batch = pending;
-                        pending = U.newLinkedHashMap(batchSize);
+                        case NEW_BATCH:
+                            needNewBatch = true;
+                            prevOperation = null;
 
-                        operation = null;
+                            break;
+
+                        default:
+                            assert false : "Unexpected result: " + addRes;
                     }
                 }
-                finally {
-                    val.writeLock().unlock();
+
+                // Process collected batch
+                applied = applyBatch(pending, true, this);
+
+                if (applied) {
+                    // Wake up awaiting writers
+                    flusherWriterLock.lock();
+
+                    try {
+                        flusherWriterCanWrite.signalAll();
+                    }
+                    finally {
+                        flusherWriterLock.unlock();
+                    }
                 }
+                else {
+                    // Return values to queue
+                    ArrayList<Map.Entry<K, StatefulValue<K,V>>> pendingList = new ArrayList(pending.entrySet());
 
-                if (batch != null && !batch.isEmpty()) {
-                    applyBatch(batch, true);
-                    batch = null;
+                    for (int i = pendingList.size() - 1; i >= 0; i--)
+                        queue.addFirst(F.t(pendingList.get(i).getKey(), pendingList.get(i).getValue()));
                 }
             }
+        }
 
-            // Process the remainder.
-            if (!pending.isEmpty())
-                applyBatch(pending, true);
+        /**
+         * Trying to add key and statefull value pairs into pending map.
+         *
+         * @param pending Map to populate.
+         * @param key Key to add.
+         * @param val Stateful value to add.
+         * @return {@code BatchingResult.ADDED} if pair was sucessfully added,
+         *     {@code BatchingResult.SKIPPED} if pair cannot be processed by this thread,
+         *     {@code BatchingResult.NEW_BATCH} if pair require new batch (pending map) to be added.
+         */
+        public BatchingResult tryAddStatefulValue(
+            Map<K, StatefulValue<K, V>> pending,
+            StoreOperation prevOperation,
+            K key,
+            StatefulValue<K, V> val
+        ) {
+            ValueStatus status = val.status();
+
+            assert !(pending.isEmpty() && prevOperation != null) : "prev operation cannot be " + prevOperation
+                + " if prev map is empty!";
+
+            if (acquired(status))
+                // Another thread is helping us, continue to the next entry.
+                return BatchingResult.SKIPPED;
+
+            if (!writeCoalescing && pending.containsKey(key))
+                return BatchingResult.NEW_BATCH;
+
+            if (status == ValueStatus.RETRY)
+                retryEntriesCnt.decrementAndGet();
+
+            assert retryEntriesCnt.get() >= 0;
+
+            if (pending.size() == batchSize)
+                return BatchingResult.NEW_BATCH;
+
+            // We scan for the next operation and apply batch on operation change. Null means new batch.
+            if (prevOperation != val.operation() && prevOperation != null)
+                // Operation is changed, so we need to perform a batch.
+                return BatchingResult.NEW_BATCH;
+            else {
+                val.status(ValueStatus.PENDING);
+
+                pending.put(key, val);
+
+                return BatchingResult.ADDED;
+            }
         }
     }
 
@@ -860,6 +1261,20 @@ Map<K, StatefulValue<K, V>> writeCache() {
         return writeCache;
     }
 
+    /**
+     * For test purposes only.
+     *
+     * @return Flusher maps for the underlying store operations.
+     */
+    Map<K, StatefulValue<K,V>>[] flusherMaps() {
+        Map<K, StatefulValue<K,V>>[] result = new Map[flushThreadCnt];
+
+        for (int i=0; i < flushThreadCnt; i++)
+            result[i] = flushThreads[i].flusherWriteMap;
+
+        return result;
+    }
+
     /**
      * Enumeration that represents possible operations on the underlying store.
      */
@@ -888,6 +1303,20 @@ private enum ValueStatus {
         FLUSHED,
     }
 
+    /**
+     * Enumeration that represents possible result of "add to batch" operation.
+     */
+    private enum BatchingResult {
+        /** Added to batch */
+        ADDED,
+
+        /** Skipped. */
+        SKIPPED,
+
+        /** Need new batch. */
+        NEW_BATCH
+    }
+
     /**
      * Checks if given status indicates pending or complete flush operation.
      *
@@ -901,6 +1330,7 @@ private boolean acquired(ValueStatus status) {
     /**
      * A state-value-operation trio.
      *
+     * @param <K> Key type.
      * @param <V> Value type.
      */
     private static class StatefulValue<K, V> extends ReentrantReadWriteLock {
@@ -949,7 +1379,7 @@ private StoreOperation operation() {
         }
 
         /**
-         * @return Value status
+         * @return Value status.
          */
         private ValueStatus status() {
             return valStatus;
@@ -980,7 +1410,7 @@ private void update(@Nullable Entry<? extends K, ? extends V> val,
         }
 
         /**
-         * Awaits a signal on flush condition
+         * Awaits a signal on flush condition.
          *
          * @throws IgniteInterruptedCheckedException If thread was interrupted.
          */
@@ -1023,4 +1453,4 @@ private void signalFlushed() {
             return S.toString(StatefulValue.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractSelfTest.java
index 323278f9c5d30..3bac90686ba33 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractSelfTest.java
@@ -20,12 +20,14 @@
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.internal.processors.cache.GridCacheTestStore;
@@ -59,16 +61,29 @@ public abstract class GridCacheWriteBehindStoreAbstractSelfTest extends GridComm
     /**
      * Initializes store.
      *
-     * @param flushThreadCnt Count of flush threads
+     * @param flushThreadCnt Count of flush threads.
      * @throws Exception If failed.
      */
     protected void initStore(int flushThreadCnt) throws Exception {
+        initStore(flushThreadCnt, CacheConfiguration.DFLT_WRITE_BEHIND_COALESCING);
+    }
+
+    /**
+     * Initializes store.
+     *
+     * @param flushThreadCnt Count of flush threads.
+     * @param writeCoalescing write coalescing flag.
+     * @throws Exception If failed.
+     */
+    protected void initStore(int flushThreadCnt, boolean writeCoalescing) throws Exception {
         store = new GridCacheWriteBehindStore<>(null, "", "", log, delegate);
 
         store.setFlushFrequency(FLUSH_FREQUENCY);
 
         store.setFlushSize(CACHE_SIZE);
 
+        store.setWriteCoalescing(writeCoalescing);
+
         store.setFlushThreadCount(flushThreadCnt);
 
         delegate.reset();
@@ -83,8 +98,11 @@ protected void initStore(int flushThreadCnt) throws Exception {
      */
     protected void shutdownStore() throws Exception {
         store.stop();
-
-        assertTrue("Store cache must be empty after shutdown", store.writeCache().isEmpty());
+        if (store.getWriteCoalescing())
+            assertTrue("Store cache must be empty after shutdown", store.writeCache().isEmpty());
+        else
+            for (Map<?,?> fMap : store.flusherMaps())
+                assertTrue("Store flusher cache must be empty after shutdown", fMap.isEmpty());
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractTest.java
index e9674f3c90b52..ca43ce3ba3d5c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractTest.java
@@ -37,6 +37,10 @@
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.transactions.Transaction;
 import org.jetbrains.annotations.Nullable;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Parameterized.Parameter;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreMultithreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreMultithreadedSelfTest.java
index bc6b7bd83f581..15c58d9f515f4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreMultithreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreMultithreadedSelfTest.java
@@ -21,19 +21,38 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
  * Multithreaded tests for {@link org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore}.
  */
 public class GridCacheWriteBehindStoreMultithreadedSelfTest extends GridCacheWriteBehindStoreAbstractSelfTest {
+    /**
+     * This test performs complex set of operations on store with coalescing from multiple threads.
+     *
+     * @throws Exception If failed.
+     */
+    public void testPutGetRemoveWithCoalescing() throws Exception {
+        testPutGetRemove(true);
+    }
+
+    /**
+     * This test performs complex set of operations on store without coalescing from multiple threads.
+     *
+     * @throws Exception If failed.
+     */
+    public void testPutGetRemoveWithoutCoalescing() throws Exception {
+        testPutGetRemove(false);
+    }
+
     /**
      * This test performs complex set of operations on store from multiple threads.
      *
      * @throws Exception If failed.
      */
-    public void testPutGetRemove() throws Exception {
-        initStore(2);
+    private void testPutGetRemove(boolean writeCoalescing) throws Exception {
+        initStore(2, writeCoalescing);
 
         Set<Integer> exp;
 
@@ -62,27 +81,55 @@ public void testPutGetRemove() throws Exception {
             assertEquals("Invalid value for key " + key, "val" + key, map.get(key));
     }
 
+    /**
+     * Tests that cache with write coalescing would keep values if underlying store fails.
+     *
+     * @throws Exception if failed.
+     */
+    public void testStoreFailureWithCoalescing() throws Exception {
+        testStoreFailure(true);
+    }
+
+    /**
+     * Tests that cache without write coalescing would keep values if underlying store fails.
+     *
+     * @throws Exception if failed.
+     */
+    public void testStoreFailureWithoutCoalescing() throws Exception {
+        testStoreFailure(false);
+    }
+
     /**
      * Tests that cache would keep values if underlying store fails.
      *
      * @throws Exception If failed.
      */
-    public void testStoreFailure() throws Exception {
+    private void testStoreFailure(boolean writeCoalescing) throws Exception {
         delegate.setShouldFail(true);
 
-        initStore(2);
+        initStore(2, writeCoalescing);
 
         Set<Integer> exp;
 
         try {
+            Thread timer = new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        U.sleep(FLUSH_FREQUENCY+50);
+                    } catch (IgniteInterruptedCheckedException e) {
+                        assertTrue("Timer was interrupted", false);
+                    }
+                    delegate.setShouldFail(false);
+                }
+            });
+            timer.start();
             exp = runPutGetRemoveMultithreaded(10, 10);
 
-            U.sleep(FLUSH_FREQUENCY);
+            timer.join();
 
             info(">>> There are " + store.getWriteBehindErrorRetryCount() + " entries in RETRY state");
 
-            delegate.setShouldFail(false);
-
             // Despite that we set shouldFail flag to false, flush thread may just have caught an exception.
             // If we move store to the stopping state right away, this value will be lost. That's why this sleep
             // is inserted here to let all exception handlers in write-behind store exit.
@@ -110,17 +157,38 @@ public void testStoreFailure() throws Exception {
             assertEquals("Invalid value for key " + key, "val" + key, map.get(key));
     }
 
+    /**
+     * Tests store (with write coalescing) consistency in case of high put rate,
+     * when flush is performed from the same thread as put or remove operation.
+     *
+     * @throws Exception If failed.
+     */
+    public void testFlushFromTheSameThreadWithCoalescing() throws Exception {
+        testFlushFromTheSameThread(true);
+    }
+
+    /**
+     * Tests store (without write coalescing) consistency in case of high put rate,
+     * when flush is performed from the same thread as put or remove operation.
+     *
+     * @throws Exception If failed.
+     */
+    public void testFlushFromTheSameThreadWithoutCoalescing() throws Exception {
+        testFlushFromTheSameThread(false);
+    }
+
     /**
      * Tests store consistency in case of high put rate, when flush is performed from the same thread
      * as put or remove operation.
      *
+     * @param writeCoalescing write coalescing flag.
      * @throws Exception If failed.
      */
-    public void testFlushFromTheSameThread() throws Exception {
+    private void testFlushFromTheSameThread(boolean writeCoalescing) throws Exception {
         // 50 milliseconds should be enough.
         delegate.setOperationDelay(50);
 
-        initStore(2);
+        initStore(2, writeCoalescing);
 
         Set<Integer> exp;
 
@@ -162,4 +230,4 @@ public void testFlushFromTheSameThread() throws Exception {
         for (Integer key : exp)
             assertEquals("Invalid value for key " + key, "val" + key, map.get(key));
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java
index 67e26ab66bd72..9a487a4268726 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java
@@ -35,11 +35,30 @@
  */
 public class GridCacheWriteBehindStoreSelfTest extends GridCacheWriteBehindStoreAbstractSelfTest {
     /**
-     * Tests correct store shutdown when underlying store fails,
+     * Tests correct store (with write coalescing) shutdown when underlying store fails.
      *
      * @throws Exception If failed.
      */
-    public void testShutdownWithFailure() throws Exception {
+    public void testShutdownWithFailureWithCoalescing() throws Exception {
+        testShutdownWithFailure(true);
+    }
+
+    /**
+     * Tests correct store (without write coalescing) shutdown when underlying store fails.
+     *
+     * @throws Exception If failed.
+     */
+    public void testShutdownWithFailureWithoutCoalescing() throws Exception {
+        testShutdownWithFailure(false);
+    }
+
+    /**
+     * Tests correct store shutdown when underlying store fails.
+     *
+     * @param writeCoalescing Write coalescing flag.
+     * @throws Exception If failed.
+     */
+    private void testShutdownWithFailure(final boolean writeCoalescing) throws Exception {
         final AtomicReference<Exception> err = new AtomicReference<>();
 
         multithreadedAsync(new Runnable() {
@@ -47,7 +66,7 @@ public void testShutdownWithFailure() throws Exception {
                 try {
                     delegate.setShouldFail(true);
 
-                    initStore(2);
+                    initStore(2, writeCoalescing);
 
                     try {
                         store.write(new CacheEntryImpl<>(1, "val1"));
@@ -70,10 +89,31 @@ public void testShutdownWithFailure() throws Exception {
     }
 
     /**
+     * Simple store (with write coalescing) test.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSimpleStoreWithCoalescing() throws Exception {
+        testSimpleStore(true);
+    }
+
+    /**
+     * Simple store (without write coalescing) test.
+     *
      * @throws Exception If failed.
      */
-    public void testSimpleStore() throws Exception {
-        initStore(2);
+    public void testSimpleStoreWithoutCoalescing() throws Exception {
+        testSimpleStore(false);
+    }
+
+    /**
+     * Simple store test.
+     *
+     * @param writeCoalescing Write coalescing flag.
+     * @throws Exception If failed.
+     */
+    private void testSimpleStore(boolean writeCoalescing) throws Exception {
+        initStore(2, writeCoalescing);
 
         try {
             store.write(new CacheEntryImpl<>(1, "v1"));
@@ -94,15 +134,36 @@ public void testSimpleStore() throws Exception {
         }
     }
 
+    /**
+     * Check that all values written to the store with coalescing will be in underlying store after timeout
+     * or due to size limits.
+     *
+     * @throws Exception If failed.
+     */
+    public void testValuePropagationWithCoalescing() throws Exception {
+        testValuePropagation(true);
+    }
+
+    /**
+     * Check that all values written to the store without coalescing will be in underlying store after timeout
+     * or due to size limits.
+     *
+     * @throws Exception If failed.
+     */
+    public void testValuePropagationWithoutCoalescing() throws Exception {
+        testValuePropagation(false);
+    }
+
     /**
      * Check that all values written to the store will be in underlying store after timeout or due to size limits.
      *
+     * @param writeCoalescing Write coalescing flag
      * @throws Exception If failed.
      */
     @SuppressWarnings({"NullableProblems"})
-    public void testValuePropagation() throws Exception {
+    private void testValuePropagation(boolean writeCoalescing) throws Exception {
         // Need to test size-based write.
-        initStore(1);
+        initStore(1, writeCoalescing);
 
         try {
             for (int i = 0; i < CACHE_SIZE * 2; i++)
@@ -131,13 +192,32 @@ public void testValuePropagation() throws Exception {
         }
     }
 
+    /**
+     * Tests store with write coalescing behaviour under continuous put of the same key with different values.
+     *
+     * @throws Exception If failed.
+     */
+    public void testContinuousPutWithCoalescing() throws Exception {
+        testContinuousPut(true);
+    }
+
+    /**
+     * Tests store without write coalescing behaviour under continuous put of the same key with different values.
+     *
+     * @throws Exception If failed.
+     */
+    public void testContinuousPutWithoutCoalescing() throws Exception {
+        testContinuousPut(false);
+    }
+
     /**
      * Tests store behaviour under continuous put of the same key with different values.
      *
-     * @throws Exception If failed
+     * @param writeCoalescing Write coalescing flag for cache.
+     * @throws Exception If failed.
      */
-    public void testContinuousPut() throws Exception {
-        initStore(2);
+    private void testContinuousPut(boolean writeCoalescing) throws Exception {
+        initStore(2, writeCoalescing);
 
         try {
             final AtomicBoolean running = new AtomicBoolean(true);
@@ -168,18 +248,23 @@ public void testContinuousPut() throws Exception {
                 }
             }, 1, "put");
 
+            U.sleep(FLUSH_FREQUENCY * 2 + 500);
+            running.set(false);
             U.sleep(FLUSH_FREQUENCY * 2 + 500);
 
             int delegatePutCnt = delegate.getPutAllCount();
 
-            running.set(false);
 
             fut.get();
 
             log().info(">>> [putCnt = " + actualPutCnt.get() + ", delegatePutCnt=" + delegatePutCnt + "]");
 
             assertTrue("No puts were made to the underlying store", delegatePutCnt > 0);
-            assertTrue("Too many puts were made to the underlying store", delegatePutCnt < actualPutCnt.get() / 10);
+            if (store.getWriteCoalescing()) {
+                assertTrue("Too many puts were made to the underlying store", delegatePutCnt < actualPutCnt.get() / 10);
+            } else {
+                assertTrue("Too few puts cnt=" + actualPutCnt.get() + " << storePutCnt=" + delegatePutCnt, delegatePutCnt > actualPutCnt.get() / 2);
+            }
         }
         finally {
             shutdownStore();
@@ -192,14 +277,35 @@ public void testContinuousPut() throws Exception {
             assertEquals("Invalid value stored", "val" + i, delegate.getMap().get(i));
     }
 
+    /**
+     * Tests that all values were put into the store with write coalescing will be written to the underlying store
+     * after shutdown is called.
+     *
+     * @throws Exception If failed.
+     */
+    public void testShutdownWithCoalescing() throws Exception {
+        testShutdown(true);
+    }
+
+    /**
+     * Tests that all values were put into the store without write coalescing will be written to the underlying store
+     * after shutdown is called.
+     *
+     * @throws Exception If failed.
+     */
+    public void testShutdownWithoutCoalescing() throws Exception {
+        testShutdown(false);
+    }
+
     /**
      * Tests that all values were put into the store will be written to the underlying store
      * after shutdown is called.
      *
+     * @param writeCoalescing Write coalescing flag.
      * @throws Exception If failed.
      */
-    public void testShutdown() throws Exception {
-        initStore(2);
+    private void testShutdown(boolean writeCoalescing) throws Exception {
+        initStore(2, writeCoalescing);
 
         try {
             final AtomicBoolean running = new AtomicBoolean(true);
@@ -241,16 +347,37 @@ public void testShutdown() throws Exception {
             assertEquals("Invalid value stored", "val" + i, delegate.getMap().get(i));
     }
 
+    /**
+     * Tests that all values will be written to the underlying store
+     * right in the same order as they were put into the store with coalescing.
+     *
+     * @throws Exception If failed.
+     */
+    public void testBatchApplyWithCoalescing() throws Exception {
+        testBatchApply(true);
+    }
+
+    /**
+     * Tests that all values will be written to the underlying store
+     * right in the same order as they were put into the store without coalescing.
+     *
+     * @throws Exception If failed.
+     */
+    public void testBatchApplyWithoutCoalescing() throws Exception {
+        testBatchApply(false);
+    }
+
     /**
      * Tests that all values will be written to the underlying store
      * right in the same order as they were put into the store.
      *
+     * @param writeCoalescing Write coalescing flag.
      * @throws Exception If failed.
      */
-    public void testBatchApply() throws Exception {
+    private void testBatchApply(boolean writeCoalescing) throws Exception {
         delegate = new GridCacheTestStore(new ConcurrentLinkedHashMap<Integer, String>());
 
-        initStore(1);
+        initStore(1, writeCoalescing);
 
         List<Integer> intList = new ArrayList<>(CACHE_SIZE);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java
new file mode 100644
index 0000000000000..8ea109dd2edfd
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.store;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicWriteOrderMode;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest;
+import org.apache.ignite.lang.IgniteBiInClosure;
+import org.apache.ignite.lang.IgniteFuture;
+
+import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+
+/**
+ * This class provides non write coalescing tests for {@link org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore}.
+ */
+public class IgnteCacheClientWriteBehindStoreNonCoalescingTest extends IgniteCacheAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheMode cacheMode() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return ATOMIC;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicWriteOrderMode atomicWriteOrderMode() {
+        return CLOCK;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected NearCacheConfiguration nearConfiguration() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Factory<CacheStore> cacheStoreFactory() {
+        return new TestIncrementStoreFactory();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNonCoalescingIncrementing() throws Exception {
+        Ignite ignite = grid(0);
+
+        IgniteCache<Integer, Integer> cache = ignite.cache(null);
+
+        assertEquals(cache.getConfiguration(CacheConfiguration.class).getCacheStoreFactory().getClass(),
+            TestIncrementStoreFactory.class);
+
+        Set<Integer> keys = new HashSet<>();
+
+        for (int i = 0; i < 1000; i++) {
+            keys.add(i);
+
+            cache.put(i, i);
+        }
+
+        Collection<IgniteFuture<?>> futs = new ArrayList<>();
+
+        for (int i = 0; i < 100; i++)
+            futs.add(updateKeys(cache, keys));
+
+        for (IgniteFuture<?> fut : futs)
+            fut.get();
+    }
+
+    /**
+     * Update specified keys in async mode.
+     *
+     * @param cache Cache to use.
+     * @param keys Keys to update.
+     * @return IgniteFuture.
+     */
+    private IgniteFuture<?>  updateKeys(IgniteCache<Integer, Integer> cache, Set<Integer> keys) {
+        IgniteCache asyncCache = cache.withAsync();
+
+        // Using EntryProcessor.invokeAll to increment every value in place.
+        asyncCache.invokeAll(keys, new EntryProcessor<Integer, Integer, Object>() {
+            @Override public Object process(MutableEntry<Integer, Integer> entry, Object... arguments)
+                throws EntryProcessorException {
+                entry.setValue(entry.getValue() + 1);
+
+                return null;
+            }
+        });
+
+        return asyncCache.future();
+    }
+
+    /**
+     * Test increment store factory.
+     */
+    public static class TestIncrementStoreFactory implements Factory<CacheStore> {
+        /** {@inheritDoc} */
+        @Override public CacheStore create() {
+            return new TestIncrementStore();
+        }
+    }
+
+    /**
+     * Test cache store to validate int value incrementing
+     */
+    public static class TestIncrementStore extends CacheStoreAdapter<Object, Object> {
+        /** {@inheritDoc} */
+        @Override public void loadCache(IgniteBiInClosure<Object, Object> clo, Object... args) {
+            for (Map.Entry<Object, Object> e : storeMap.entrySet())
+                clo.apply(e.getKey(), e.getValue());
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object load(Object key) {
+            return storeMap.get(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Object, ? extends Object> entry) {
+            Object oldValue = storeMap.put(entry.getKey(), entry.getValue());
+
+            if (oldValue instanceof Integer && entry.getValue() instanceof Integer) {
+                Integer oldInt = (Integer)oldValue;
+                Integer newInt = (Integer)entry.getValue();
+
+                assertTrue(
+                    "newValue(" + newInt + ") != oldValue(" + oldInt + ")+1 !",
+                    newInt == oldInt + 1);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) {
+            storeMap.remove(key);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheWriteBehindTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheWriteBehindTestSuite.java
index b4cdfa8a76a20..dff93ffa5ecf0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheWriteBehindTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheWriteBehindTestSuite.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStoreReplicatedTest;
 import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStoreSelfTest;
 import org.apache.ignite.internal.processors.cache.store.IgnteCacheClientWriteBehindStoreAtomicTest;
+import org.apache.ignite.internal.processors.cache.store.IgnteCacheClientWriteBehindStoreNonCoalescingTest;
 import org.apache.ignite.internal.processors.cache.store.IgnteCacheClientWriteBehindStoreTxTest;
 
 /**
@@ -49,6 +50,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(GridCachePartitionedWritesTest.class));
         suite.addTest(new TestSuite(IgnteCacheClientWriteBehindStoreAtomicTest.class));
         suite.addTest(new TestSuite(IgnteCacheClientWriteBehindStoreTxTest.class));
+        suite.addTest(new TestSuite(IgnteCacheClientWriteBehindStoreNonCoalescingTest.class));
 
         return suite;
     }

From c48322b79cafb561211501813c805aebd4b85540 Mon Sep 17 00:00:00 2001
From: Valentin Kulichenko <valentin.lulichenko@gmail.com>
Date: Thu, 13 Apr 2017 11:29:30 +0300
Subject: [PATCH 133/516] IGNITE-4954 - Configurable expiration timeout for
 Cassandra session. This closes #1785.

---
 .../cassandra/datasource/DataSource.java      | 50 +++++++++++++------
 .../session/CassandraSessionImpl.java         | 23 +++++----
 .../cassandra/session/pool/SessionPool.java   |  6 +--
 .../session/pool/SessionWrapper.java          | 15 +++---
 4 files changed, 62 insertions(+), 32 deletions(-)

diff --git a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
index 1ba3c7d238328..754d902ac9392 100644
--- a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
+++ b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
@@ -17,6 +17,16 @@
 
 package org.apache.ignite.cache.store.cassandra.datasource;
 
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
 import com.datastax.driver.core.AuthProvider;
 import com.datastax.driver.core.Cluster;
 import com.datastax.driver.core.ConsistencyLevel;
@@ -31,25 +41,13 @@
 import com.datastax.driver.core.policies.ReconnectionPolicy;
 import com.datastax.driver.core.policies.RetryPolicy;
 import com.datastax.driver.core.policies.SpeculativeExecutionPolicy;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.io.Serializable;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.UUID;
-
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.store.cassandra.session.CassandraSession;
 import org.apache.ignite.cache.store.cassandra.session.CassandraSessionImpl;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
  * Data source abstraction to specify configuration of the Cassandra session to be used.
@@ -64,6 +62,9 @@ public class DataSource implements Externalizable {
      */
     private static final UUID NULL_OBJECT = UUID.fromString("45ffae47-3193-5910-84a2-048fe65735d9");
 
+    /** Default expiration timeout for Cassandra driver session. */
+    public static final long DFLT_SESSION_EXPIRATION_TIMEOUT = 300000; // 5 minutes.
+
     /** Number of rows to immediately fetch in CQL statement execution. */
     private Integer fetchSize;
 
@@ -141,6 +142,9 @@ public class DataSource implements Externalizable {
     /** Netty options to use for connection. */
     private NettyOptions nettyOptions;
 
+    /** Expiration timeout for Cassandra driver session. */
+    private long sessionExpirationTimeout = DFLT_SESSION_EXPIRATION_TIMEOUT;
+
     /** Cassandra session wrapper instance. */
     private volatile CassandraSession ses;
 
@@ -459,6 +463,23 @@ public void setNettyOptions(NettyOptions options) {
         invalidate();
     }
 
+    /**
+     * Sets expiration timeout for Cassandra driver session. Idle sessions that are not
+     * used during this timeout value will be automatically closed and recreated later
+     * on demand.
+     * <p>
+     * If set to {@code 0}, timeout is disabled.
+     * <p>
+     * Default value is {@link #DFLT_SESSION_EXPIRATION_TIMEOUT}.
+     *
+     * @param sessionExpirationTimeout Expiration timeout for Cassandra driver session.
+     */
+    public void setSessionExpirationTimeout(long sessionExpirationTimeout) {
+        this.sessionExpirationTimeout = sessionExpirationTimeout;
+
+        invalidate();
+    }
+
     /**
      * Creates Cassandra session wrapper if it wasn't created yet and returns it
      *
@@ -541,7 +562,8 @@ public synchronized CassandraSession session(IgniteLogger log) {
         if (nettyOptions != null)
             builder = builder.withNettyOptions(nettyOptions);
 
-        return ses = new CassandraSessionImpl(builder, fetchSize, readConsistency, writeConsistency, log);
+        return ses = new CassandraSessionImpl(
+            builder, fetchSize, readConsistency, writeConsistency, sessionExpirationTimeout, log);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java
index ac116865c1be5..19b88c988dc6c 100644
--- a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java
+++ b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java
@@ -17,6 +17,13 @@
 
 package org.apache.ignite.cache.store.cassandra.session;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.Cache;
 import com.datastax.driver.core.BatchStatement;
 import com.datastax.driver.core.BoundStatement;
 import com.datastax.driver.core.Cluster;
@@ -30,13 +37,6 @@
 import com.datastax.driver.core.exceptions.AlreadyExistsException;
 import com.datastax.driver.core.exceptions.InvalidQueryException;
 import com.datastax.driver.core.querybuilder.Batch;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.cache.Cache;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.store.cassandra.common.CassandraHelper;
@@ -83,6 +83,9 @@ public class CassandraSessionImpl implements CassandraSession {
     /** Consistency level for Cassandra WRITE operations (insert/update/delete). */
     private ConsistencyLevel writeConsistency;
 
+    /** Expiration timeout. */
+    private long expirationTimeout;
+
     /** Logger. */
     private IgniteLogger log;
 
@@ -102,11 +105,12 @@ public class CassandraSessionImpl implements CassandraSession {
      * @param log Logger.
      */
     public CassandraSessionImpl(Cluster.Builder builder, Integer fetchSize, ConsistencyLevel readConsistency,
-        ConsistencyLevel writeConsistency, IgniteLogger log) {
+        ConsistencyLevel writeConsistency, long expirationTimeout, IgniteLogger log) {
         this.builder = builder;
         this.fetchSize = fetchSize;
         this.readConsistency = readConsistency;
         this.writeConsistency = writeConsistency;
+        this.expirationTimeout = expirationTimeout;
         this.log = log;
     }
 
@@ -504,7 +508,8 @@ else if (CassandraHelper.isPreparedStatementClusterError(e))
     /** {@inheritDoc} */
     @Override public synchronized void close() throws IOException {
         if (decrementSessionRefs() == 0 && ses != null) {
-            SessionPool.put(this, ses);
+            SessionPool.put(this, ses, expirationTimeout);
+
             ses = null;
         }
     }
diff --git a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java
index 95938bd7740a1..4de8516d95b32 100644
--- a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java
+++ b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionPool.java
@@ -17,13 +17,13 @@
 
 package org.apache.ignite.cache.store.cassandra.session.pool;
 
-import com.datastax.driver.core.Session;
 import java.lang.Thread.State;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import com.datastax.driver.core.Session;
 import org.apache.ignite.cache.store.cassandra.session.CassandraSessionImpl;
 
 /**
@@ -98,14 +98,14 @@ private static class SessionMonitor extends Thread {
      * @param cassandraSes Session wrapper.
      * @param driverSes Driver session.
      */
-    public static void put(CassandraSessionImpl cassandraSes, Session driverSes) {
+    public static void put(CassandraSessionImpl cassandraSes, Session driverSes, long expirationTimeout) {
         if (cassandraSes == null || driverSes == null)
             return;
 
         SessionWrapper old;
 
         synchronized (sessions) {
-            old = sessions.put(cassandraSes, new SessionWrapper(driverSes));
+            old = sessions.put(cassandraSes, new SessionWrapper(driverSes, expirationTimeout));
 
             if (monitorSingleton == null || State.TERMINATED.equals(monitorSingleton.getState())) {
                 monitorSingleton = new SessionMonitor();
diff --git a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java
index 7c5722bb8f21f..68b9dd480f8b8 100644
--- a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java
+++ b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/pool/SessionWrapper.java
@@ -24,12 +24,12 @@
  * Wrapper for Cassandra driver session, responsible for monitoring session expiration and its closing.
  */
 public class SessionWrapper {
-    /** Expiration timeout for Cassandra driver session. */
-    public static final long DFLT_EXPIRATION_TIMEOUT = 300000;  // 5 minutes.
-
     /** Cassandra driver session. */
     private Session ses;
 
+    /** Expiration timeout. */
+    private long expirationTimeout;
+
     /** Wrapper creation time.  */
     private long time;
 
@@ -38,9 +38,11 @@ public class SessionWrapper {
      *
      * @param ses Cassandra driver session.
      */
-    public SessionWrapper(Session ses) {
+    public SessionWrapper(Session ses, long expirationTimeout) {
         this.ses = ses;
-        this.time = System.currentTimeMillis();
+        this.expirationTimeout = expirationTimeout;
+
+        time = System.currentTimeMillis();
     }
 
     /**
@@ -49,7 +51,7 @@ public SessionWrapper(Session ses) {
      * @return true if session expired.
      */
     public boolean expired() {
-        return System.currentTimeMillis() - time > DFLT_EXPIRATION_TIMEOUT;
+        return expirationTimeout > 0 && System.currentTimeMillis() - time > expirationTimeout;
     }
 
     /**
@@ -66,6 +68,7 @@ public Session driverSession() {
      */
     public void release() {
         CassandraHelper.closeSession(ses);
+
         ses = null;
     }
 }

From 1c1a2f3e86c091bbbd2668db1cc7d72ff13d91ec Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 19 Apr 2017 13:27:49 +0300
Subject: [PATCH 134/516] Fix merge ignite-1.8.5-p1

---
 .../query/h2/sql/GridSqlQueryParser.java      |  2 +-
 .../query/h2/sql/GridQueryParsingTest.java    | 40 ++-----------------
 2 files changed, 5 insertions(+), 37 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index 2b086584017f0..0f940e9c3846c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -1053,7 +1053,7 @@ private GridSqlElement parseExpression0(Expression expression, boolean calcTypes
 
             GridSqlOperation res = new GridSqlOperation(EXISTS);
 
-            res.addChild(new GridSqlSubquery(parse(qry, null)));
+            res.addChild(parseQueryExpression(qry));
 
             return res;
         }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
index 9dcc80bc649b6..477451a1cd255 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
@@ -289,37 +289,6 @@ public void testParseSelectAndUnion() throws Exception {
         checkQuery("select sch.\"#\".* from Person \"#\"");
     }
 
-    /**
-     * Query AST transformation heavily depends on this behavior.
-     *
-     * @throws Exception If failed.
-     */
-    public void testParseTableFilter() throws Exception {
-        Prepared prepared = parse("select Person.old, p1.old from Person, Person p1");
-
-        GridSqlSelect select = (GridSqlSelect)new GridSqlQueryParser(false).parse(prepared);
-
-        GridSqlJoin join = (GridSqlJoin)select.from();
-
-        GridSqlTable tbl1 = (GridSqlTable)join.leftTable();
-        GridSqlAlias tbl2Alias = (GridSqlAlias)join.rightTable();
-        GridSqlTable tbl2 = tbl2Alias.child();
-
-        // Must be distinct objects, even if it is the same table.
-        assertNotSame(tbl1, tbl2);
-        assertNotNull(tbl1.dataTable());
-        assertNotNull(tbl2.dataTable());
-        assertSame(tbl1.dataTable(), tbl2.dataTable());
-
-        GridSqlColumn col1 = (GridSqlColumn)select.column(0);
-        GridSqlColumn col2 = (GridSqlColumn)select.column(1);
-
-        assertSame(tbl1, col1.expressionInFrom());
-
-        // Alias in FROM must be included in column.
-        assertSame(tbl2Alias, col2.expressionInFrom());
-    }
-
     /**
      * Query AST transformation heavily depends on this behavior.
      *
@@ -329,7 +298,7 @@ public void testParseTableFilter() throws Exception {
         Prepared prepared = parse("select Person.old, p1.old, p1.addrId from Person, Person p1 " +
             "where exists(select 1 from Address a where a.id = p1.addrId)");
 
-        GridSqlSelect select = (GridSqlSelect)new GridSqlQueryParser().parse(prepared);
+        GridSqlSelect select = (GridSqlSelect)new GridSqlQueryParser(false).parse(prepared);
 
         GridSqlJoin join = (GridSqlJoin)select.from();
 
@@ -338,8 +307,7 @@ public void testParseTableFilter() throws Exception {
         GridSqlTable tbl2 = tbl2Alias.child();
 
         // Must be distinct objects, even if it is the same table.
-        //assertNotSame(tbl1, tbl2);
-
+        assertNotSame(tbl1, tbl2);
         assertNotNull(tbl1.dataTable());
         assertNotNull(tbl2.dataTable());
         assertSame(tbl1.dataTable(), tbl2.dataTable());
@@ -353,9 +321,9 @@ public void testParseTableFilter() throws Exception {
         assertSame(tbl2Alias, col2.expressionInFrom());
 
         // In EXISTS we must correctly reference the column from the outer query.
-        GridSqlElement exists = select.where();
+        GridSqlAst exists = select.where();
         GridSqlSubquery subqry = exists.child();
-        GridSqlSelect subSelect = (GridSqlSelect)subqry.select();
+        GridSqlSelect subSelect = subqry.child();
 
         GridSqlColumn p1AddrIdCol = (GridSqlColumn)select.column(2);
 

From ea3e625fe1269383d46d0be55adc85662a216792 Mon Sep 17 00:00:00 2001
From: Alexander Belyak <alexandr.belyak@xored.com>
Date: Wed, 19 Apr 2017 13:29:01 +0300
Subject: [PATCH 135/516] IGNITE-4927 Use write behind coalescing from
 CacheConfiguration in GridCacheStoreManagerAdapter to configure actual cache
 store.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../processors/cache/GridCacheAttributes.java |  7 +++
 .../processors/cache/GridCacheProcessor.java  |  4 ++
 .../store/GridCacheStoreManagerAdapter.java   |  1 +
 .../utils/PlatformConfigurationUtils.java     |  2 +
 ...StoreSessionWriteBehindCoalescingTest.java | 47 +++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |  2 +
 6 files changed, 63 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java
index e17b91a0414f9..57f12f635489b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java
@@ -302,6 +302,13 @@ public int writeBehindBatchSize() {
         return ccfg.getWriteBehindBatchSize();
     }
 
+    /**
+     * @return Write-behind cache store write coalescing.
+     */
+    public boolean writeBehindCoalescing() {
+        return ccfg.getWriteBehindCoalescing();
+    }
+
     /**
      * @return Interceptor class name.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 50e1379961298..fe7da4d14e537 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2927,6 +2927,10 @@ private void checkCache(CacheConfiguration locCfg, CacheConfiguration rmtCfg, Cl
                 "Write behind batch size", locAttr.writeBehindBatchSize(), rmtAttr.writeBehindBatchSize(),
                 false);
 
+            CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "writeBehindCacheCoalescing",
+                "Write behind coalescing", locAttr.writeBehindCoalescing(), rmtAttr.writeBehindCoalescing(),
+                false);
+
             CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "writeBehindEnabled",
                 "Write behind enabled", locAttr.writeBehindEnabled(), rmtAttr.writeBehindEnabled(), false);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index a83018682109d..e9e90ea3c25d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -181,6 +181,7 @@ private CacheStore cacheStoreWrapper(GridKernalContext ctx,
         store.setFlushThreadCount(cfg.getWriteBehindFlushThreadCount());
         store.setFlushFrequency(cfg.getWriteBehindFlushFrequency());
         store.setBatchSize(cfg.getWriteBehindBatchSize());
+        store.setWriteCoalescing(cfg.getWriteBehindCoalescing());
 
         return store;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
index c0fde976d72ed..daec38b9bfaf9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
@@ -166,6 +166,7 @@ public static CacheConfiguration readCacheConfiguration(BinaryRawReaderEx in) {
         ccfg.setSqlOnheapRowCacheSize(in.readInt());
         ccfg.setStartSize(in.readInt());
         ccfg.setWriteBehindBatchSize(in.readInt());
+        ccfg.setWriteBehindCoalescing(in.readBoolean());
         ccfg.setWriteBehindEnabled(in.readBoolean());
         ccfg.setWriteBehindFlushFrequency(in.readLong());
         ccfg.setWriteBehindFlushSize(in.readInt());
@@ -791,6 +792,7 @@ public static void writeCacheConfiguration(BinaryRawWriter writer, CacheConfigur
         writer.writeInt(ccfg.getSqlOnheapRowCacheSize());
         writer.writeInt(ccfg.getStartSize());
         writer.writeInt(ccfg.getWriteBehindBatchSize());
+        writer.writeBoolean(ccfg.getWriteBehindCoalescing());
         writer.writeBoolean(ccfg.isWriteBehindEnabled());
         writer.writeLong(ccfg.getWriteBehindFlushFrequency());
         writer.writeInt(ccfg.getWriteBehindFlushSize());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java
new file mode 100644
index 0000000000000..c15b2f4e2c334
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.integration;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ * Integration test write behind cache store with {@link CacheConfiguration#getWriteBehindCoalescing()}={@code False}
+ * parameter.
+ */
+public class IgniteCacheTxStoreSessionWriteBehindCoalescingTest extends IgniteCacheStoreSessionWriteBehindAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return TRANSACTIONAL;
+    }
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration result = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = result.getCacheConfiguration()[0];
+
+        ccfg.setWriteBehindCoalescing(false);
+
+        return result;
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 9fcf31a322f2a..501d23e754517 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -140,6 +140,7 @@
 import org.apache.ignite.internal.processors.cache.integration.IgniteCacheTxNoReadThroughTest;
 import org.apache.ignite.internal.processors.cache.integration.IgniteCacheTxNoWriteThroughTest;
 import org.apache.ignite.internal.processors.cache.integration.IgniteCacheTxStoreSessionTest;
+import org.apache.ignite.internal.processors.cache.integration.IgniteCacheTxStoreSessionWriteBehindCoalescingTest;
 import org.apache.ignite.internal.processors.cache.integration.IgniteCacheTxStoreSessionWriteBehindTest;
 import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryLocalAtomicSwapDisabledSelfTest;
 import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryLocalTransactionalSelfTest;
@@ -181,6 +182,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheAtomicStoreSessionTest.class);
         suite.addTestSuite(IgniteCacheTxStoreSessionTest.class);
         suite.addTestSuite(IgniteCacheAtomicStoreSessionWriteBehindTest.class);
+        suite.addTestSuite(IgniteCacheTxStoreSessionWriteBehindCoalescingTest.class);
         suite.addTestSuite(IgniteCacheTxStoreSessionWriteBehindTest.class);
 
         suite.addTestSuite(IgniteCacheAtomicNoReadThroughTest.class);

From c8155fd8c2b12ca69fd5877fd21327ec746e1225 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dgovorukhin@gridgain.com>
Date: Fri, 21 Apr 2017 16:23:00 +0300
Subject: [PATCH 136/516] ignite-1.9.2 backport of ignite-4949

---
 .../store/jdbc/CacheAbstractJdbcStore.java    |  66 +++++----
 .../cache/store/CacheOsStoreManager.java      |   3 +
 .../store/GridCacheStoreManagerAdapter.java   |   9 ++
 .../CacheJdbcPojoStoreAbstractSelfTest.java   |  19 ++-
 ...naryMarshallerStoreKeepBinarySelfTest.java |  28 ++++
 ...rStoreKeepBinaryWithSqlEscapeSelfTest.java |  28 ++++
 .../store/jdbc/CacheJdbcPojoStoreTest.java    | 136 ++++++++++++++----
 .../testframework/junits/IgniteMock.java      |   4 +
 8 files changed, 233 insertions(+), 60 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
index 7af238b1bc112..57bf311d2a9d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
@@ -48,7 +48,6 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.cache.CacheTypeFieldMetadata;
 import org.apache.ignite.cache.CacheTypeMetadata;
 import org.apache.ignite.cache.store.CacheStore;
@@ -61,6 +60,7 @@
 import org.apache.ignite.cache.store.jdbc.dialect.OracleDialect;
 import org.apache.ignite.cache.store.jdbc.dialect.SQLServerDialect;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.binary.BinaryEnumObjectImpl;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
@@ -556,22 +556,35 @@ private JdbcTypeField[] translateFields(Collection<CacheTypeFieldMetadata> oldFl
 
     /**
      * @param type Type name to check.
+     * @param binarySupported True if binary marshaller enable.
      * @return {@code True} if class not found.
      */
-    protected TypeKind kindForName(String type) {
+    protected TypeKind kindForName(String type, boolean binarySupported) {
         if (BUILT_IN_TYPES.contains(type))
             return TypeKind.BUILT_IN;
 
+        if (binarySupported)
+            return TypeKind.BINARY;
+
         try {
             Class.forName(type);
 
             return TypeKind.POJO;
         }
-        catch(ClassNotFoundException ignored) {
-            return TypeKind.BINARY;
+        catch (ClassNotFoundException e) {
+            throw new CacheException("Can not find class " + type +
+                ", check your classPath or try to use BinaryMarshaller", e);
         }
     }
 
+    /**
+     * @param type Type name to check.
+     * @return {@code True} if class not found.
+     */
+    protected TypeKind kindForName(String type) {
+        return kindForName(type, ignite.configuration().getMarshaller() instanceof BinaryMarshaller);
+    }
+
     /**
      * @param cacheName Cache name to check mappings for.
      * @return Type mappings for specified cache name.
@@ -636,11 +649,7 @@ private Map<Object, EntryMapping> getOrCreateCacheMappings(@Nullable String cach
                     String keyType = type.getKeyType();
                     String valType = type.getValueType();
 
-                    TypeKind keyKind = kindForName(keyType);
-
-                    if (!binarySupported && keyKind == TypeKind.BINARY)
-                        throw new CacheException("Key type has no class [cache=" + U.maskName(cacheName) +
-                            ", type=" + keyType + "]");
+                    TypeKind keyKind = kindForName(keyType, binarySupported);
 
                     checkTypeConfiguration(cacheName, keyKind, keyType, type.getKeyFields());
 
@@ -650,21 +659,11 @@ private Map<Object, EntryMapping> getOrCreateCacheMappings(@Nullable String cach
                         throw new CacheException("Key type must be unique in type metadata [cache=" +
                             U.maskName(cacheName) + ", type=" + keyType + "]");
 
-                    TypeKind valKind = kindForName(valType);
+                    TypeKind valKind = kindForName(valType, binarySupported);
 
                     checkTypeConfiguration(cacheName, valKind, valType, type.getValueFields());
 
                     entryMappings.put(keyTypeId, new EntryMapping(cacheName, dialect, type, keyKind, valKind, sqlEscapeAll));
-
-                    // Add one more binding to binary typeId for POJOs,
-                    // because object could be passed to store in binary format.
-                    if (binarySupported && keyKind == TypeKind.POJO) {
-                        keyTypeId = typeIdForTypeName(TypeKind.BINARY, keyType);
-
-                        valKind = valKind == TypeKind.POJO ? TypeKind.BINARY : valKind;
-
-                        entryMappings.put(keyTypeId, new EntryMapping(cacheName, dialect, type, TypeKind.BINARY, valKind, sqlEscapeAll));
-                    }
                 }
 
                 Map<String, Map<Object, EntryMapping>> mappings = new HashMap<>(cacheMappings);
@@ -1398,10 +1397,17 @@ protected void fillParameter(PreparedStatement stmt, int idx, JdbcTypeField fiel
                             // No-op.
                     }
                 }
-                else if (field.getJavaFieldType().isEnum() && fieldVal instanceof Enum) {
-                    Enum val = (Enum)fieldVal;
+                else if (field.getJavaFieldType().isEnum()) {
+                    if (fieldVal instanceof Enum) {
+                        Enum val = (Enum)fieldVal;
 
-                    fieldVal = NUMERIC_TYPES.contains(field.getDatabaseFieldType()) ? val.ordinal() : val.name();
+                        fieldVal = NUMERIC_TYPES.contains(field.getDatabaseFieldType()) ? val.ordinal() : val.name();
+                    }
+                    else if (fieldVal instanceof BinaryEnumObjectImpl) {
+                        BinaryEnumObjectImpl val = (BinaryEnumObjectImpl)fieldVal;
+
+                        fieldVal = val.enumOrdinal();
+                    }
                 }
 
                 stmt.setObject(idx, fieldVal);
@@ -1453,14 +1459,14 @@ protected int fillKeyParameters(PreparedStatement stmt, EntryMapping m, Object k
      */
     protected int fillValueParameters(PreparedStatement stmt, int idx, EntryMapping em, Object val)
         throws CacheWriterException {
-        TypeKind valKind = em.valueKind();
-
-        // Object could be passed by cache in binary format in case of cache configured with setStoreKeepBinary(true).
-        if (valKind == TypeKind.POJO && val instanceof BinaryObject)
-            valKind = TypeKind.BINARY;
-
         for (JdbcTypeField field : em.uniqValFlds) {
-            Object fieldVal = extractParameter(em.cacheName, em.valueType(), valKind, field.getJavaFieldName(), val);
+            Object fieldVal = extractParameter(
+                em.cacheName,
+                em.valueType(),
+                em.valueKind(),
+                field.getJavaFieldName(),
+                val
+            );
 
             fillParameter(stmt, idx++, field, fieldVal);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java
index 27771ff3e629a..2e23d04f735d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java
@@ -77,6 +77,9 @@ public CacheOsStoreManager(GridKernalContext ctx, CacheConfiguration cfg) {
 
     /** {@inheritDoc} */
     @Override public boolean convertBinary() {
+        if (alwaysKeepBinary)
+            return false;
+
         return configuredConvertBinary() && !(cfgStore instanceof PlatformCacheStore);
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index e9e90ea3c25d8..814166d7dafac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -31,10 +31,13 @@
 import javax.cache.integration.CacheLoaderException;
 import javax.cache.integration.CacheWriterException;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cache.store.CacheStoreSession;
 import org.apache.ignite.cache.store.CacheStoreSessionListener;
+import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
@@ -113,6 +116,9 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
     /** */
     private boolean globalSesLsnrs;
 
+    /** Always keep binary. */
+    protected boolean alwaysKeepBinary;
+
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public void initialize(@Nullable CacheStore cfgStore, Map sesHolders) throws IgniteCheckedException {
@@ -148,6 +154,9 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
         sesHolder = sesHolder0;
 
         locStore = U.hasAnnotation(cfgStore, CacheLocalStore.class);
+
+        if (cfgStore instanceof CacheJdbcPojoStore)
+            alwaysKeepBinary = true;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
index e890109574156..d6d235221448c 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ConnectorConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -221,6 +222,7 @@ protected CacheConfiguration cacheConfiguration() throws Exception {
         cc.setAtomicityMode(transactional ? TRANSACTIONAL : ATOMIC);
         cc.setSwapEnabled(false);
         cc.setWriteBehindEnabled(false);
+        cc.setStoreKeepBinary(storeKeepBinary());
 
         CacheJdbcPojoStoreFactory<Object, Object> storeFactory = new CacheJdbcPojoStoreFactory<>();
         storeFactory.setDialect(new H2Dialect());
@@ -237,6 +239,13 @@ protected CacheConfiguration cacheConfiguration() throws Exception {
         return cc;
     }
 
+    /**
+     * @return Flag indicate keep value in binary format or not.
+     */
+    protected boolean storeKeepBinary(){
+        return false;
+    }
+
     /**
      * Fill in-memory database with sample data.
      *
@@ -397,6 +406,8 @@ public void testLoadCachePrimitiveKeysTx() throws Exception {
      * @throws Exception If failed.
      */
     private void checkPutRemove() throws Exception {
+        boolean binaryMarshaller = marshaller() instanceof BinaryMarshaller || marshaller() == null;
+
         IgniteCache<Object, Person> c1 = grid().cache(CACHE_NAME);
 
         Connection conn = getConnection();
@@ -429,7 +440,9 @@ private void checkPutRemove() throws Exception {
             assertEquals(-2, rs.getInt(2));
             assertEquals(testDate, rs.getDate(3));
             assertEquals("Person-to-test-put-insert", rs.getString(4));
-            assertEquals(testGender.toString(), rs.getString(5));
+
+            assertEquals(testGender.toString(),
+                binaryMarshaller ? Gender.values()[rs.getInt(5)].toString(): rs.getString(5));
 
             assertFalse("Unexpected more data in result set", rs.next());
 
@@ -448,7 +461,9 @@ private void checkPutRemove() throws Exception {
             assertEquals(-3, rs.getInt(2));
             assertEquals(testDate, rs.getDate(3));
             assertEquals("Person-to-test-put-update", rs.getString(4));
-            assertEquals(testGender.toString(), rs.getString(5));
+
+            assertEquals(testGender.toString(),
+                binaryMarshaller ? Gender.values()[rs.getInt(5)].toString(): rs.getString(5));
 
             assertFalse("Unexpected more data in result set", rs.next());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java
new file mode 100644
index 0000000000000..dfca8647e2840
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.jdbc;
+
+/**
+ *
+ */
+public class CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest extends CacheJdbcPojoStoreBinaryMarshallerSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean storeKeepBinary() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java
new file mode 100644
index 0000000000000..c7e1f794ab208
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.jdbc;
+
+/**
+ *
+ */
+public class CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest extends CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean storeKeepBinary() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
index 4a0b1daf646d1..bb85cabbd25a5 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.cache.store.jdbc;
 
+import java.lang.reflect.Field;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
@@ -28,18 +29,22 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import javax.cache.integration.CacheWriterException;
-
+import org.apache.ignite.Ignite;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
 import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect;
 import org.apache.ignite.cache.store.jdbc.model.Organization;
 import org.apache.ignite.cache.store.jdbc.model.OrganizationKey;
 import org.apache.ignite.cache.store.jdbc.model.Person;
 import org.apache.ignite.cache.store.jdbc.model.PersonComplexKey;
 import org.apache.ignite.cache.store.jdbc.model.PersonKey;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.config.GridTestProperties;
 import org.apache.ignite.testframework.junits.cache.GridAbstractCacheStoreSelfTest;
 import org.h2.jdbcx.JdbcConnectionPool;
 
@@ -56,6 +61,12 @@ public class CacheJdbcPojoStoreTest extends GridAbstractCacheStoreSelfTest<Cache
     /** Person count. */
     protected static final int PERSON_CNT = 100000;
 
+    /** Ignite. */
+    private Ignite ig;
+
+    /** Binary enable. */
+    private boolean binaryEnable;
+
     /**
      * @throws Exception If failed.
      */
@@ -225,6 +236,12 @@ public CacheJdbcPojoStoreTest() throws Exception {
         U.closeQuiet(conn);
 
         super.beforeTest();
+
+        Ignite ig = U.field(store, "ignite");
+
+        this.ig = ig;
+
+        binaryEnable = ig.configuration().getMarshaller() instanceof BinaryMarshaller;
     }
 
     /**
@@ -289,26 +306,48 @@ public void testLoadCache() throws Exception {
 
         U.closeQuiet(conn);
 
-        final Collection<OrganizationKey> orgKeys = new ConcurrentLinkedQueue<>();
-        final Collection<PersonKey> prnKeys = new ConcurrentLinkedQueue<>();
-        final Collection<PersonComplexKey> prnComplexKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> orgKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> prnKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> prnComplexKeys = new ConcurrentLinkedQueue<>();
 
         IgniteBiInClosure<Object, Object> c = new CI2<Object, Object>() {
             @Override public void apply(Object k, Object v) {
-                if (k instanceof OrganizationKey && v instanceof Organization)
-                    orgKeys.add((OrganizationKey)k);
-                else if (k instanceof PersonKey && v instanceof Person)
-                    prnKeys.add((PersonKey)k);
-                else if (k instanceof PersonComplexKey && v instanceof Person) {
-                    PersonComplexKey key = (PersonComplexKey)k;
-
-                    Person val = (Person)v;
-
-                    assertTrue("Key ID should be the same as value ID", key.getId() == val.getId());
-                    assertTrue("Key orgID should be the same as value orgID", key.getOrgId() == val.getOrgId());
-                    assertEquals("name" + key.getId(), val.getName());
-
-                    prnComplexKeys.add((PersonComplexKey)k);
+                if (binaryEnable){
+                    if (k instanceof BinaryObject && v instanceof BinaryObject) {
+                        BinaryObject key = (BinaryObject)k;
+                        BinaryObject val = (BinaryObject)v;
+
+                        String keyType = key.type().typeName();
+                        String valType = val.type().typeName();
+
+                        if (OrganizationKey.class.getName().equals(keyType)
+                            && Organization.class.getName().equals(valType))
+                            orgKeys.add(key);
+
+                        if (PersonKey.class.getName().equals(keyType)
+                            && Person.class.getName().equals(valType))
+                            prnKeys.add(key);
+
+                        if (PersonComplexKey.class.getName().equals(keyType)
+                            && Person.class.getName().equals(valType))
+                            prnComplexKeys.add(key);
+                    }
+                }else {
+                    if (k instanceof OrganizationKey && v instanceof Organization)
+                        orgKeys.add(k);
+                    else if (k instanceof PersonKey && v instanceof Person)
+                        prnKeys.add(k);
+                    else if (k instanceof PersonComplexKey && v instanceof Person) {
+                        PersonComplexKey key = (PersonComplexKey)k;
+
+                        Person val = (Person)v;
+
+                        assertTrue("Key ID should be the same as value ID", key.getId() == val.getId());
+                        assertTrue("Key orgID should be the same as value orgID", key.getOrgId() == val.getOrgId());
+                        assertEquals("name" + key.getId(), val.getName());
+
+                        prnComplexKeys.add(k);
+                    }
                 }
             }
         };
@@ -319,15 +358,16 @@ else if (k instanceof PersonComplexKey && v instanceof Person) {
         assertEquals(PERSON_CNT, prnKeys.size());
         assertEquals(PERSON_CNT, prnComplexKeys.size());
 
-        Collection<OrganizationKey> tmpOrgKeys = new ArrayList<>(orgKeys);
-        Collection<PersonKey> tmpPrnKeys = new ArrayList<>(prnKeys);
-        Collection<PersonComplexKey> tmpPrnComplexKeys = new ArrayList<>(prnComplexKeys);
+        Collection<Object> tmpOrgKeys = new ArrayList<>(orgKeys);
+        Collection<Object> tmpPrnKeys = new ArrayList<>(prnKeys);
+        Collection<Object> tmpPrnComplexKeys = new ArrayList<>(prnComplexKeys);
 
         orgKeys.clear();
         prnKeys.clear();
         prnComplexKeys.clear();
 
-        store.loadCache(c, OrganizationKey.class.getName(), "SELECT name, city, id FROM ORGANIZATION",
+        store.loadCache(
+            c, OrganizationKey.class.getName(), "SELECT name, city, id FROM ORGANIZATION",
             PersonKey.class.getName(), "SELECT org_id, id, name FROM Person WHERE id < 1000");
 
         assertEquals(ORGANIZATION_CNT, orgKeys.size());
@@ -377,14 +417,29 @@ public void testParallelLoad() throws Exception {
 
         U.closeQuiet(conn);
 
-        final Collection<PersonComplexKey> prnComplexKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> prnComplexKeys = new ConcurrentLinkedQueue<>();
 
         IgniteBiInClosure<Object, Object> c = new CI2<Object, Object>() {
             @Override public void apply(Object k, Object v) {
-                if (k instanceof PersonComplexKey && v instanceof Person)
-                    prnComplexKeys.add((PersonComplexKey)k);
-                else
-                    fail("Unexpected entry [key=" + k + ", value=" + v + "]");
+                if (binaryEnable) {
+                    if (k instanceof BinaryObject && v instanceof BinaryObject) {
+                        BinaryObject key = (BinaryObject)k;
+                        BinaryObject val = (BinaryObject)v;
+
+                        String keyType = key.type().typeName();
+                        String valType = val.type().typeName();
+
+                        if (PersonComplexKey.class.getName().equals(keyType)
+                            && Person.class.getName().equals(valType))
+                            prnComplexKeys.add(key);
+                    }
+                }
+                else {
+                    if (k instanceof PersonComplexKey && v instanceof Person)
+                        prnComplexKeys.add(k);
+                    else
+                        fail("Unexpected entry [key=" + k + ", value=" + v + "]");
+                }
             }
         };
 
@@ -437,7 +492,7 @@ public void testWriteRetry() throws Exception {
         ses.newSession(null);
 
         try {
-            store.write(new CacheEntryImpl<>(k1, v1));
+            store.write(new CacheEntryImpl<>(wrap(k1), wrap(v1)));
 
             fail("CacheWriterException wasn't thrown.");
         }
@@ -466,4 +521,29 @@ public void testTimestamp() throws Exception {
 
         assertNull(store.load(k));
     }
+
+    /**
+     * @param obj Object.
+     */
+    private Object wrap(Object obj) throws IllegalAccessException {
+        if (binaryEnable) {
+            Class<?> cls = obj.getClass();
+
+            BinaryObjectBuilder builder = ig.binary().builder(cls.getName());
+
+            for (Field f : cls.getDeclaredFields()) {
+                if (f.getName().contains("serialVersionUID"))
+                    continue;
+
+                f.setAccessible(true);
+
+                builder.setField(f.getName(), f.get(obj));
+            }
+
+            return builder.build();
+        }
+
+        return obj;
+    }
+
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
index 5722fa3d6056a..14ba030e24590 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.binary.BinaryCachingMetadataHandler;
 import org.apache.ignite.internal.binary.BinaryContext;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl;
 import org.apache.ignite.internal.processors.cacheobject.NoOpBinary;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -327,6 +328,9 @@ public IgniteMock(
                     return typeName.hashCode();
                 }
             };
+
+            if (marshaller instanceof BinaryMarshaller)
+                ctx.configure((BinaryMarshaller)marshaller, configuration());
         }
 
         binaryMock = new NoOpBinary() {

From 8cae6dfd7ed000d06b37ad53f5d04aa3fd534d66 Mon Sep 17 00:00:00 2001
From: Alexander Belyak <alexandr.belyak@xored.com>
Date: Fri, 21 Apr 2017 16:53:05 +0300
Subject: [PATCH 137/516] IGNITE-4927 Use write behind coalescing from
 CacheConfiguration in GridCacheStoreManagerAdapter to configure actual cache
 store.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../store/GridCacheWriteBehindStore.java      | 20 ++++---
 ...heStoreSessionWriteBehindAbstractTest.java | 56 ++++++++++++-----
 ...StoreSessionWriteBehindCoalescingTest.java | 60 ++++++++++++++++---
 3 files changed, 104 insertions(+), 32 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
index 4a1b11e807bf3..8537aaba69282 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
@@ -385,8 +385,10 @@ public int getWriteBehindErrorRetryCount() {
             if (log.isDebugEnabled())
                 log.debug("Stopping write-behind store for cache '" + cacheName + '\'');
 
-            for (Flusher f : flushThreads) {
-                if (!f.isEmpty())
+            if (writeCoalescing)
+                wakeUp();
+            else {
+                for (Flusher f : flushThreads)
                     f.wakeUp();
             }
 
@@ -897,7 +899,7 @@ protected Flusher(String gridName,
             IgniteLogger log) {
             super(gridName, name, log);
 
-            flusherCacheCriticalSize = cacheCriticalSize/flushThreadCnt;
+            flusherCacheCriticalSize = cacheCriticalSize / flushThreadCnt;
 
             assert flusherCacheCriticalSize > batchSize;
 
@@ -930,7 +932,9 @@ private void putToFlusherWriteCache(
             throws IgniteInterruptedCheckedException {
             assert !writeCoalescing : "Unexpected write coalescing.";
 
-            if (queue.sizex() > flusherCacheCriticalSize) {
+            int qSize = queue.sizex();
+
+            if (qSize > flusherCacheCriticalSize) {
                 while (queue.sizex() > flusherCacheCriticalSize) {
                     wakeUp();
 
@@ -960,6 +964,8 @@ private void putToFlusherWriteCache(
 
                 cacheTotalOverflowCntr.incrementAndGet();
             }
+            else if (qSize > batchSize)
+                wakeUp();
 
             queue.add(F.t(key, newVal));
 
@@ -1055,6 +1061,9 @@ private void awaitOperationsAvailableNonCoalescing() throws InterruptedException
                     if (queue.sizex() >= batchSize)
                         return;
 
+                    if (stopping.get())
+                        return;
+
                     if (cacheFlushFreq > 0)
                         LockSupport.parkNanos(cacheFlushFreqNanos);
                     else
@@ -1065,9 +1074,6 @@ private void awaitOperationsAvailableNonCoalescing() throws InterruptedException
 
                     if (Thread.interrupted())
                         throw new InterruptedException();
-
-                    if (stopping.get())
-                        return;
                 }
             }
             finally {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java
index bbb6f4c68e921..4b0f7c0bb22c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java
@@ -48,6 +48,9 @@ public abstract class IgniteCacheStoreSessionWriteBehindAbstractTest extends Ign
     /** */
     private static volatile CountDownLatch latch;
 
+    /** */
+    protected static volatile CountDownLatch entLatch;
+
     /** */
     private static volatile ExpectedData expData;
 
@@ -66,32 +69,38 @@ public abstract class IgniteCacheStoreSessionWriteBehindAbstractTest extends Ign
         return null;
     }
 
-    /** {@inheritDoc} */
+    /**
+     * @param igniteInstanceName Ignite instance name.
+     * @return Cache configuration.
+     * @throws Exception In case of error.
+     */
     @SuppressWarnings("unchecked")
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
+        CacheConfiguration ccfg0 = super.cacheConfiguration(igniteInstanceName);
 
-        assert cfg.getCacheConfiguration().length == 1;
-
-        CacheConfiguration ccfg0 = cfg.getCacheConfiguration()[0];
 
         ccfg0.setReadThrough(true);
         ccfg0.setWriteThrough(true);
         ccfg0.setWriteBehindBatchSize(10);
         ccfg0.setWriteBehindFlushSize(10);
-        ccfg0.setWriteBehindFlushFrequency(60_000);
+        ccfg0.setWriteBehindFlushFrequency(600);
         ccfg0.setWriteBehindEnabled(true);
 
         ccfg0.setCacheStoreFactory(singletonFactory(new TestStore()));
 
-        CacheConfiguration ccfg1 = cacheConfiguration(gridName);
+        return ccfg0;
+    }
 
-        ccfg1.setReadThrough(true);
-        ccfg1.setWriteThrough(true);
-        ccfg1.setWriteBehindBatchSize(10);
-        ccfg1.setWriteBehindFlushSize(10);
-        ccfg1.setWriteBehindFlushFrequency(60_000);
-        ccfg1.setWriteBehindEnabled(true);
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        assert cfg.getCacheConfiguration().length == 1;
+
+        CacheConfiguration ccfg0 = cacheConfiguration(igniteInstanceName);
+
+        CacheConfiguration ccfg1 = cacheConfiguration(igniteInstanceName);
 
         ccfg1.setName(CACHE_NAME1);
 
@@ -120,6 +129,7 @@ private void testCache(String cacheName) throws Exception {
 
         try {
             latch = new CountDownLatch(2);
+            entLatch = new CountDownLatch(11);
 
             expData = new ExpectedData("writeAll", cacheName);
 
@@ -127,13 +137,17 @@ private void testCache(String cacheName) throws Exception {
                 cache.put(i, i);
 
             assertTrue(latch.await(10_000, TimeUnit.MILLISECONDS));
+
+            assertTrue(entLatch.await(10_000,TimeUnit.MILLISECONDS));
         }
         finally {
             latch = null;
+            entLatch = null;
         }
 
         try {
             latch = new CountDownLatch(2);
+            entLatch = new CountDownLatch(11);
 
             expData = new ExpectedData("deleteAll", cacheName);
 
@@ -141,16 +155,20 @@ private void testCache(String cacheName) throws Exception {
                 cache.remove(i);
 
             assertTrue(latch.await(10_000, TimeUnit.MILLISECONDS));
+
+            assertTrue(entLatch.await(10_000,TimeUnit.MILLISECONDS));
         }
         finally {
             latch = null;
+            entLatch = null;
         }
     }
 
     /**
      *
      */
-    private class TestStore implements CacheStore<Object, Object> {
+    protected class TestStore implements CacheStore<Object, Object> {
+
         /** Auto-injected store session. */
         @CacheStoreSessionResource
         private CacheStoreSession ses;
@@ -195,6 +213,9 @@ private class TestStore implements CacheStore<Object, Object> {
             assertTrue("Unexpected entries: " + entries, entries.size() == 10 || entries.size() == 1);
 
             checkSession("writeAll");
+
+            for (int i = 0; i < entries.size(); i++)
+                entLatch.countDown();
         }
 
         /** {@inheritDoc} */
@@ -209,6 +230,9 @@ private class TestStore implements CacheStore<Object, Object> {
             assertTrue("Unexpected keys: " + keys, keys.size() == 10 || keys.size() == 1);
 
             checkSession("deleteAll");
+
+            for (int i = 0; i < keys.size(); i++)
+                entLatch.countDown();
         }
 
         /**
@@ -221,7 +245,7 @@ private CacheStoreSession session() {
         /**
          * @param mtd Called stored method.
          */
-        private void checkSession(String mtd) {
+        protected void checkSession(String mtd) {
             assertNotNull(ignite);
 
             CacheStoreSession ses = session();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java
index c15b2f4e2c334..ad7b7c2bf50c2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java
@@ -17,11 +17,13 @@
 
 package org.apache.ignite.internal.processors.cache.integration;
 
-import org.apache.ignite.IgniteCache;
+import java.util.Collection;
+import javax.cache.Cache;
+import javax.cache.integration.CacheWriterException;
 import org.apache.ignite.cache.CacheAtomicityMode;
-import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 
 /**
  * Integration test write behind cache store with {@link CacheConfiguration#getWriteBehindCoalescing()}={@code False}
@@ -33,15 +35,55 @@ public class IgniteCacheTxStoreSessionWriteBehindCoalescingTest extends IgniteCa
         return TRANSACTIONAL;
     }
 
-    /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration result = super.getConfiguration(gridName);
 
-        CacheConfiguration ccfg = result.getCacheConfiguration()[0];
+    /**
+     * @param igniteInstanceName Ignite instance name.
+     * @return Cache configuration.
+     * @throws Exception In case of error.
+     */
+    @SuppressWarnings("unchecked")
+    protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
+        CacheConfiguration ccfg = super.cacheConfiguration(igniteInstanceName);
 
         ccfg.setWriteBehindCoalescing(false);
 
-        return result;
+        ccfg.setCacheStoreFactory(singletonFactory(new TestNonCoalescingStore()));
+
+        return ccfg;
+    }
+
+    /**
+     *
+     */
+    private class TestNonCoalescingStore extends TestStore {
+
+        /** {@inheritDoc} */
+        @Override public void writeAll(Collection<Cache.Entry<?, ?>> entries) throws CacheWriterException {
+            log.info("writeAll: " + entries);
+
+            assertTrue("Unexpected entries: " + entries, entries.size() <= 10);
+
+            checkSession("writeAll");
+
+            for (int i = 0; i < entries.size(); i++)
+                entLatch.countDown();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            fail();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void deleteAll(Collection<?> keys) throws CacheWriterException {
+            log.info("deleteAll: " + keys);
+
+            assertTrue("Unexpected keys: " + keys, keys.size() <= 10);
+
+            checkSession("deleteAll");
+
+            for (int i = 0; i < keys.size(); i++)
+                entLatch.countDown();
+        }
     }
 }
\ No newline at end of file

From fde6d4427131f4b74fcdf88f1caae4f57d225a87 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Thu, 20 Apr 2017 19:39:45 +0300
Subject: [PATCH 138/516] IGNITE-5043 .NET:
 CacheConfiguration.WriteBehindCoalescing

(cherry picked from commit 217c6be)
---
 .../utils/PlatformConfigurationUtils.java     |  4 +-
 .../Cache/CacheConfigurationTest.cs           |  3 +
 .../IgniteConfigurationSerializerTest.cs      |  4 +-
 .../Cache/Configuration/CacheConfiguration.cs | 62 +++++++++++++------
 .../IgniteConfigurationSection.xsd            |  7 +++
 5 files changed, 59 insertions(+), 21 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
index daec38b9bfaf9..5ca85e8c4143c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
@@ -166,11 +166,11 @@ public static CacheConfiguration readCacheConfiguration(BinaryRawReaderEx in) {
         ccfg.setSqlOnheapRowCacheSize(in.readInt());
         ccfg.setStartSize(in.readInt());
         ccfg.setWriteBehindBatchSize(in.readInt());
-        ccfg.setWriteBehindCoalescing(in.readBoolean());
         ccfg.setWriteBehindEnabled(in.readBoolean());
         ccfg.setWriteBehindFlushFrequency(in.readLong());
         ccfg.setWriteBehindFlushSize(in.readInt());
         ccfg.setWriteBehindFlushThreadCount(in.readInt());
+        ccfg.setWriteBehindCoalescing(in.readBoolean());
         ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.fromOrdinal(in.readInt()));
         ccfg.setReadThrough(in.readBoolean());
         ccfg.setWriteThrough(in.readBoolean());
@@ -792,11 +792,11 @@ public static void writeCacheConfiguration(BinaryRawWriter writer, CacheConfigur
         writer.writeInt(ccfg.getSqlOnheapRowCacheSize());
         writer.writeInt(ccfg.getStartSize());
         writer.writeInt(ccfg.getWriteBehindBatchSize());
-        writer.writeBoolean(ccfg.getWriteBehindCoalescing());
         writer.writeBoolean(ccfg.isWriteBehindEnabled());
         writer.writeLong(ccfg.getWriteBehindFlushFrequency());
         writer.writeInt(ccfg.getWriteBehindFlushSize());
         writer.writeInt(ccfg.getWriteBehindFlushThreadCount());
+        writer.writeBoolean(ccfg.getWriteBehindCoalescing());
         writeEnumInt(writer, ccfg.getWriteSynchronizationMode());
         writer.writeBoolean(ccfg.isReadThrough());
         writer.writeBoolean(ccfg.isWriteThrough());
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs
index fb8725c59b221..aea5989c56b2e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs
@@ -212,6 +212,8 @@ private static void AssertConfigIsDefault(CacheConfiguration cfg)
             Assert.AreEqual(CacheConfiguration.DefaultWriteBehindEnabled, cfg.WriteBehindEnabled);
             Assert.AreEqual(CacheConfiguration.DefaultWriteBehindFlushFrequency, cfg.WriteBehindFlushFrequency);
             Assert.AreEqual(CacheConfiguration.DefaultWriteBehindFlushSize, cfg.WriteBehindFlushSize);
+            Assert.AreEqual(CacheConfiguration.DefaultWriteBehindFlushThreadCount, cfg.WriteBehindFlushThreadCount);
+            Assert.AreEqual(CacheConfiguration.DefaultWriteBehindCoalescing, cfg.WriteBehindCoalescing);
         }
 
         /// <summary>
@@ -520,6 +522,7 @@ private static CacheConfiguration GetCustomCacheConfiguration(string name = null
                 CacheStoreFactory = new CacheStoreFactoryTest(),
                 ReadThrough = true,
                 WriteThrough = true,
+                WriteBehindCoalescing = false,
                 QueryEntities = new[]
                 {
                     new QueryEntity
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs
index 26e04a9ba34e2..8bf518b415955 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs
@@ -91,7 +91,7 @@ public void TestPredefinedXml()
                                 <iLifecycleBean type='Apache.Ignite.Core.Tests.IgniteConfigurationSerializerTest+LifecycleBean' foo='15' />
                             </lifecycleBeans>
                             <cacheConfiguration>
-                                <cacheConfiguration cacheMode='Replicated' readThrough='true' writeThrough='true' enableStatistics='true'>
+                                <cacheConfiguration cacheMode='Replicated' readThrough='true' writeThrough='true' enableStatistics='true' writeBehindCoalescing='false'>
                                     <queryEntities>    
                                         <queryEntity keyType='System.Int32' valueType='System.String'>    
                                             <fields>
@@ -162,6 +162,7 @@ public void TestPredefinedXml()
             Assert.IsTrue(cacheCfg.WriteThrough);
             Assert.IsInstanceOf<MyPolicyFactory>(cacheCfg.ExpiryPolicyFactory);
             Assert.IsTrue(cacheCfg.EnableStatistics);
+            Assert.IsFalse(cacheCfg.WriteBehindCoalescing);
 
             var queryEntity = cacheCfg.QueryEntities.Single();
             Assert.AreEqual(typeof(int), queryEntity.KeyType);
@@ -645,6 +646,7 @@ private static IgniteConfiguration GetTestConfig()
                         WriteBehindFlushFrequency = TimeSpan.FromSeconds(55),
                         WriteBehindFlushSize = 66,
                         WriteBehindFlushThreadCount = 2,
+                        WriteBehindCoalescing = false,
                         WriteSynchronizationMode = CacheWriteSynchronizationMode.FullAsync,
                         NearConfiguration = new NearCacheConfiguration
                         {
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
index 2795111d66ca3..69befad2e4d23 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
@@ -150,6 +150,9 @@ public class CacheConfiguration
         /// <summary> Default value for write-through behavior. </summary>
         public const bool DefaultWriteThrough = false;
 
+        /// <summary> Default value for <see cref="WriteBehindCoalescing"/>. </summary>
+        public const bool DefaultWriteBehindCoalescing = true;
+
         /// <summary>
         /// Gets or sets the cache name.
         /// </summary>
@@ -202,6 +205,7 @@ public CacheConfiguration(string name)
             WriteBehindFlushFrequency = DefaultWriteBehindFlushFrequency;
             WriteBehindFlushSize = DefaultWriteBehindFlushSize;
             WriteBehindFlushThreadCount= DefaultWriteBehindFlushThreadCount;
+            WriteBehindCoalescing = DefaultWriteBehindCoalescing;
         }
 
         /// <summary>
@@ -269,6 +273,7 @@ internal CacheConfiguration(IBinaryRawReader reader)
             WriteBehindFlushFrequency = reader.ReadLongAsTimespan();
             WriteBehindFlushSize = reader.ReadInt();
             WriteBehindFlushThreadCount = reader.ReadInt();
+            WriteBehindCoalescing = reader.ReadBoolean();
             WriteSynchronizationMode = (CacheWriteSynchronizationMode) reader.ReadInt();
             ReadThrough = reader.ReadBoolean();
             WriteThrough = reader.ReadBoolean();
@@ -326,6 +331,7 @@ internal void Write(IBinaryRawWriter writer)
             writer.WriteLong((long) WriteBehindFlushFrequency.TotalMilliseconds);
             writer.WriteInt(WriteBehindFlushSize);
             writer.WriteInt(WriteBehindFlushThreadCount);
+            writer.WriteBoolean(WriteBehindCoalescing);
             writer.WriteInt((int) WriteSynchronizationMode);
             writer.WriteBoolean(ReadThrough);
             writer.WriteBoolean(WriteThrough);
@@ -382,11 +388,11 @@ internal void Validate(ILogger log)
         public CacheWriteSynchronizationMode WriteSynchronizationMode { get; set; }
 
         /// <summary>
-        /// Gets or sets flag indicating whether eviction is synchronized between primary, backup and near nodes.        
+        /// Gets or sets flag indicating whether eviction is synchronized between primary, backup and near nodes.
         /// If this parameter is true and swap is disabled then <see cref="ICache{TK,TV}.LocalEvict"/>
-        /// will involve all nodes where an entry is kept.  
-        /// If this property is set to false then eviction is done independently on different cache nodes.        
-        /// Note that it's not recommended to set this value to true if cache store is configured since it will allow 
+        /// will involve all nodes where an entry is kept.
+        /// If this property is set to false then eviction is done independently on different cache nodes.
+        /// Note that it's not recommended to set this value to true if cache store is configured since it will allow
         /// to significantly improve cache performance.
         /// </summary>
         [DefaultValue(DefaultEvictSynchronized)]
@@ -399,12 +405,12 @@ internal void Validate(ILogger log)
         public int EvictSynchronizedKeyBufferSize { get; set; }
 
         /// <summary>
-        /// Gets or sets concurrency level for synchronized evictions. 
-        /// This flag only makes sense with <see cref="EvictSynchronized"/> set to true. 
-        /// When synchronized evictions are enabled, it is possible that local eviction policy will try 
-        /// to evict entries faster than evictions can be synchronized with backup or near nodes. 
-        /// This value specifies how many concurrent synchronous eviction sessions should be allowed 
-        /// before the system is forced to wait and let synchronous evictions catch up with the eviction policy.       
+        /// Gets or sets concurrency level for synchronized evictions.
+        /// This flag only makes sense with <see cref="EvictSynchronized"/> set to true.
+        /// When synchronized evictions are enabled, it is possible that local eviction policy will try
+        /// to evict entries faster than evictions can be synchronized with backup or near nodes.
+        /// This value specifies how many concurrent synchronous eviction sessions should be allowed
+        /// before the system is forced to wait and let synchronous evictions catch up with the eviction policy.
         /// </summary>
         [DefaultValue(DefaultEvictSynchronizedConcurrencyLevel)]
         public int EvictSynchronizedConcurrencyLevel { get; set; }
@@ -416,11 +422,11 @@ internal void Validate(ILogger log)
         public TimeSpan EvictSynchronizedTimeout { get; set; }
 
         /// <summary>
-        /// This value denotes the maximum size of eviction queue in percents of cache size 
+        /// This value denotes the maximum size of eviction queue in percents of cache size
         /// in case of distributed cache (replicated and partitioned) and using synchronized eviction
-        /// <para/>        
-        /// That queue is used internally as a buffer to decrease network costs for synchronized eviction. 
-        /// Once queue size reaches specified value all required requests for all entries in the queue 
+        /// <para/>
+        /// That queue is used internally as a buffer to decrease network costs for synchronized eviction.
+        /// Once queue size reaches specified value all required requests for all entries in the queue
         /// are sent to remote nodes and the queue is cleared.
         /// </summary>
         [DefaultValue(DefaultMaxEvictionOverflowRatio)]
@@ -562,7 +568,7 @@ internal void Validate(ILogger log)
         /// <summary>
         /// Maximum batch size for write-behind cache store operations. 
         /// Store operations (get or remove) are combined in a batch of this size to be passed to 
-        /// <see cref="ICacheStore.WriteAll"/> or <see cref="ICacheStore.DeleteAll"/> methods. 
+        /// <see cref="ICacheStore.WriteAll"/> or <see cref="ICacheStore.DeleteAll"/> methods.
         /// </summary>
         [DefaultValue(DefaultWriteBehindBatchSize)]
         public int WriteBehindBatchSize { get; set; }
@@ -596,8 +602,8 @@ internal void Validate(ILogger log)
 
         /// <summary>
         /// Gets or sets maximum amount of memory available to off-heap storage. Possible values are
-        /// -1 means that off-heap storage is disabled. 0 means that Ignite will not limit off-heap storage 
-        /// (it's up to user to properly add and remove entries from cache to ensure that off-heap storage 
+        /// -1 means that off-heap storage is disabled. 0 means that Ignite will not limit off-heap storage
+        /// (it's up to user to properly add and remove entries from cache to ensure that off-heap storage
         /// does not grow indefinitely.
         /// Any positive value specifies the limit of off-heap storage in bytes.
         /// </summary>
@@ -700,7 +706,7 @@ internal void Validate(ILogger log)
         public IAffinityFunction AffinityFunction { get; set; }
 
         /// <summary>
-        /// Gets or sets the factory for <see cref="IExpiryPolicy"/> to be used for all cache operations, 
+        /// Gets or sets the factory for <see cref="IExpiryPolicy"/> to be used for all cache operations,
         /// unless <see cref="ICache{TK,TV}.WithExpiryPolicy"/> is called.
         /// <para />
         /// Default is null, which means no expiration.
@@ -712,5 +718,25 @@ internal void Validate(ILogger log)
         /// These statistics can be retrieved via <see cref="ICache{TK,TV}.GetMetrics()"/>.
         /// </summary>
         public bool EnableStatistics { get; set; }
+
+        /// <summary>
+        /// Gets or sets the plugin configurations.
+        /// </summary>
+        [SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
+        public ICollection<ICachePluginConfiguration> PluginConfigurations { get; set; }
+
+        /// <summary>
+        /// Gets or sets the name of the <see cref="MemoryPolicyConfiguration"/> for this cache.
+        /// See <see cref="IgniteConfiguration.MemoryConfiguration"/>.
+        /// </summary>
+        public string MemoryPolicyName { get; set; }
+
+        /// <summary>
+        /// Gets or sets write coalescing flag for write-behind cache store operations.
+        /// Store operations (get or remove) with the same key are combined or coalesced to single,
+        /// resulting operation to reduce pressure to underlying cache store.
+        /// </summary>
+        [DefaultValue(DefaultWriteBehindCoalescing)]
+        public bool WriteBehindCoalescing { get; set; }
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd
index d54a2002bdc7b..5b512a7ce9527 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd
@@ -653,6 +653,13 @@
                                             </xs:documentation>
                                         </xs:annotation>
                                     </xs:attribute>
+                                    <xs:attribute name="writeBehindCoalescing" type="xs:boolean">
+                                        <xs:annotation>
+                                            <xs:documentation>
+                                                Coalescing flag for write-behind cache store operations. Store operations (get or remove) with the same key are combined or coalesced to single, resulting operation to reduce pressure to underlying cache store.
+                                            </xs:documentation>
+                                        </xs:annotation>
+                                    </xs:attribute>
                                     <xs:attribute name="writeBehindBatchSize" type="xs:int">
                                         <xs:annotation>
                                             <xs:documentation>

From 474add7c79eb5c57b2f88ca71ceb58ed3e5cb20b Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 24 Apr 2017 14:52:07 +0300
Subject: [PATCH 139/516] IGNITE-5043 .NET:
 CacheConfiguration.WriteBehindCoalescing

Fix merge.
---
 .../Cache/Configuration/CacheConfiguration.cs        | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
index 69befad2e4d23..332ab2bd7d4c9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
@@ -719,18 +719,6 @@ internal void Validate(ILogger log)
         /// </summary>
         public bool EnableStatistics { get; set; }
 
-        /// <summary>
-        /// Gets or sets the plugin configurations.
-        /// </summary>
-        [SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
-        public ICollection<ICachePluginConfiguration> PluginConfigurations { get; set; }
-
-        /// <summary>
-        /// Gets or sets the name of the <see cref="MemoryPolicyConfiguration"/> for this cache.
-        /// See <see cref="IgniteConfiguration.MemoryConfiguration"/>.
-        /// </summary>
-        public string MemoryPolicyName { get; set; }
-
         /// <summary>
         /// Gets or sets write coalescing flag for write-behind cache store operations.
         /// Store operations (get or remove) with the same key are combined or coalesced to single,

From a4c70f12305c15cbf15a6e5eb7c500f883a8b317 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Thu, 20 Apr 2017 20:45:58 +0300
Subject: [PATCH 140/516] ignite-5041 NPE in deadlock detection fixed

---
 .../cache/DynamicCacheDescriptor.java         |  24 ++
 .../cache/GridCacheSharedContext.java         |  16 ++
 .../cache/transactions/IgniteTxManager.java   |  21 +-
 .../cache/transactions/TxDeadlock.java        |  19 +-
 .../cache/transactions/TxLocksResponse.java   |  37 +--
 ...adlockDetectionMessageMarshallingTest.java | 116 +++++++++
 ...xDeadlockDetectionUnmasrhalErrorsTest.java | 225 ++++++++++++++++++
 .../transactions/TxDeadlockNpeClientTest.java | 220 +++++++++++++++++
 .../TxDeadlockDetectionTestSuite.java         |   4 +
 9 files changed, 655 insertions(+), 27 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionUnmasrhalErrorsTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
index 8b62f03cb8649..1d0346a04e42a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
@@ -20,9 +20,11 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.plugin.CachePluginManager;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -81,6 +83,12 @@ public class DynamicCacheDescriptor {
     /** */
     private AffinityTopologyVersion rcvdFromVer;
 
+    /** Mutex. */
+    private final Object mux = new Object();
+
+    /** Cached object context for marshalling issues when cache isn't started. */
+    private volatile CacheObjectContext objCtx;
+
     /**
      * @param ctx Context.
      * @param cacheCfg Cache configuration.
@@ -209,6 +217,22 @@ public CacheConfiguration cacheConfiguration() {
         return cacheCfg;
     }
 
+    /**
+     * Creates and caches cache object context if needed.
+     *
+     * @param proc Object processor.
+     */
+    public CacheObjectContext cacheObjectContext(IgniteCacheObjectProcessor proc) throws IgniteCheckedException {
+        if (objCtx == null) {
+            synchronized (mux) {
+                if (objCtx == null)
+                    objCtx = proc.contextForCache(cacheCfg);
+            }
+        }
+
+        return objCtx;
+    }
+
     /**
      * @return Cache plugin manager.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index 117a5c33189d7..da5ff2e2714fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -392,6 +392,22 @@ public GridCacheContext<K, V> cacheContext(int cacheId) {
         return ctxMap.get(cacheId);
     }
 
+    /**
+     * Returns cache object context if created or creates new and caches it until cache started.
+     *
+     * @param cacheId Cache id.
+     */
+    public @Nullable CacheObjectContext cacheObjectContext(int cacheId) throws IgniteCheckedException {
+        GridCacheContext<K, V> ctx = ctxMap.get(cacheId);
+
+        if (ctx != null)
+            return ctx.cacheObjectContext();
+
+        DynamicCacheDescriptor desc = cache().cacheDescriptor(cacheId);
+
+        return desc != null ? desc.cacheObjectContext(kernalContext().cacheObjects()) : null;
+    }
+
     /**
      * @return Grid name.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 24f8ea918b07a..1ba0102408eb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -31,7 +31,6 @@
 import java.util.concurrent.ConcurrentMap;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteClientDisconnectedException;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryObjectException;
 import org.apache.ignite.cluster.ClusterNode;
@@ -2479,11 +2478,18 @@ private class DeadlockDetectionListener implements GridMessageListener {
         @Override public void onMessage(UUID nodeId, Object msg) {
             GridCacheMessage cacheMsg = (GridCacheMessage)msg;
 
-            unmarshall(nodeId, cacheMsg);
+            Throwable err = null;
 
-            if (cacheMsg.classError() != null) {
+            try {
+                unmarshall(nodeId, cacheMsg);
+            }
+            catch (Exception e) {
+                err = e;
+            }
+
+            if (err != null || cacheMsg.classError() != null) {
                 try {
-                    processFailedMessage(nodeId, cacheMsg);
+                    processFailedMessage(nodeId, cacheMsg, err);
                 }
                 catch(Throwable e){
                     U.error(log, "Failed to process message [senderId=" + nodeId +
@@ -2536,7 +2542,7 @@ else if (msg instanceof TxLocksResponse) {
          * @param nodeId Node ID.
          * @param msg Message.
          */
-        private void processFailedMessage(UUID nodeId, GridCacheMessage msg) throws IgniteCheckedException {
+        private void processFailedMessage(UUID nodeId, GridCacheMessage msg, Throwable err) throws IgniteCheckedException {
             switch (msg.directType()) {
                 case -24: {
                     TxLocksRequest req = (TxLocksRequest)msg;
@@ -2568,7 +2574,10 @@ private void processFailedMessage(UUID nodeId, GridCacheMessage msg) throws Igni
                         return;
                     }
 
-                    fut.onResult(nodeId, res);
+                    if (err == null)
+                        fut.onResult(nodeId, res);
+                    else
+                        fut.onDone(null, err);
                 }
 
                 break;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlock.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlock.java
index f8130e11a2abd..97db6988052c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlock.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlock.java
@@ -21,11 +21,10 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
@@ -133,11 +132,21 @@ public String toString(GridCacheSharedContext ctx) {
         for (Map.Entry<IgniteTxKey, String> e : keyLabels.entrySet()) {
             IgniteTxKey txKey = e.getKey();
 
-            GridCacheContext cctx = ctx.cacheContext(txKey.cacheId());
+            try {
+                CacheObjectContext objCtx = ctx.cacheObjectContext(txKey.cacheId());
 
-            Object val = CU.value(txKey.key(), cctx, true);
+                Object val = txKey.key().value(objCtx, true);
 
-            sb.append(e.getValue()).append(" [key=").append(val).append(", cache=").append(cctx.namexx()).append("]\n");
+                sb.append(e.getValue())
+                    .append(" [key=")
+                    .append(val)
+                    .append(", cache=")
+                    .append(objCtx.cacheName())
+                    .append("]\n");
+            }
+            catch (Exception ex) {
+                sb.append("Unable to unmarshall deadlock information for key [key=").append(e.getValue()).append("]\n");
+            }
         }
 
         return sb.toString();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java
index fa6afdd0b8a8e..a2f3037afaa08 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java
@@ -181,31 +181,36 @@ public void addKey(IgniteTxKey key) {
 
     /** {@inheritDoc} */
     @Override public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr) throws IgniteCheckedException {
-        super.finishUnmarshal(ctx, ldr);
+        try {
+            super.finishUnmarshal(ctx, ldr);
 
-        if (nearTxKeysArr != null) {
-            for (int i = 0; i < nearTxKeysArr.length; i++) {
-                IgniteTxKey key = nearTxKeysArr[i];
+            if (nearTxKeysArr != null) {
+                for (int i = 0; i < nearTxKeysArr.length; i++) {
+                    IgniteTxKey txKey = nearTxKeysArr[i];
 
-                key.finishUnmarshal(ctx.cacheContext(key.cacheId()), ldr);
+                    txKey.key().finishUnmarshal(ctx.cacheObjectContext(txKey.cacheId()), ldr);
 
-                txLocks().put(key, locksArr[i]);
+                    txLocks().put(txKey, locksArr[i]);
+                }
+
+                nearTxKeysArr = null;
+                locksArr = null;
             }
 
-            nearTxKeysArr = null;
-            locksArr = null;
-        }
+            if (txKeysArr != null) {
+                txKeys = U.newHashSet(txKeysArr.length);
 
-        if (txKeysArr != null) {
-            txKeys = U.newHashSet(txKeysArr.length);
+                for (IgniteTxKey txKey : txKeysArr) {
+                    txKey.key().finishUnmarshal(ctx.cacheObjectContext(txKey.cacheId()), ldr);
 
-            for (IgniteTxKey key : txKeysArr) {
-                key.finishUnmarshal(ctx.cacheContext(key.cacheId()), ldr);
+                    txKeys.add(txKey);
+                }
 
-                txKeys.add(key);
+                txKeysArr = null;
             }
-
-            txKeysArr = null;
+        }
+        catch (Exception e) {
+            throw new IgniteCheckedException(e);
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
new file mode 100644
index 0000000000000..eafd09f2fdbf9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.managers.communication.GridMessageListener;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class TxDeadlockDetectionMessageMarshallingTest extends GridCommonAbstractTest {
+    /** Topic. */
+    private static final String TOPIC = "mytopic";
+
+    /** Client mode. */
+    private static boolean clientMode;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(clientMode);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMessageUnmarshallWithoutCacheContext() throws Exception {
+        try {
+            Ignite ignite = startGrid(0);
+
+            CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>();
+
+            IgniteCache<Integer, Integer> cache = ignite.getOrCreateCache(ccfg);
+
+            clientMode = true;
+
+            Ignite client = startGrid(1);
+
+            final GridCacheSharedContext<Object, Object> clientCtx = ((IgniteKernal)client).context().cache().context();
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            final AtomicBoolean res = new AtomicBoolean();
+
+            clientCtx.gridIO().addMessageListener(TOPIC, new GridMessageListener() {
+                @Override public void onMessage(UUID nodeId, Object msg) {
+                    if (msg instanceof TxLocksResponse) {
+                        try {
+                            ((TxLocksResponse)msg).finishUnmarshal(clientCtx, clientCtx.deploy().globalLoader());
+
+                            res.set(true);
+                        }
+                        catch (Exception e) {
+                            log.error("Message unmarshal failed", e);
+                        }
+                        finally {
+                            latch.countDown();
+                        }
+                    }
+                }
+            });
+
+            GridCacheContext cctx = ((IgniteCacheProxy)cache).context();
+
+            KeyCacheObject key = cctx.toCacheKeyObject(1);
+
+            TxLocksResponse msg = new TxLocksResponse();
+            msg.addKey(cctx.txKey(key));
+
+            msg.prepareMarshal(cctx.shared());
+
+            ((IgniteKernal)ignite).context().cache().context().gridIO().send(
+                ((IgniteKernal)client).getLocalNodeId(), TOPIC, msg, GridIoPolicy.PUBLIC_POOL);
+
+            boolean await = latch.await(1, TimeUnit.SECONDS);
+
+            assertTrue(await && res.get());
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionUnmasrhalErrorsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionUnmasrhalErrorsTest.java
new file mode 100644
index 0000000000000..598725e328f5f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionUnmasrhalErrorsTest.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionDeadlockException;
+import org.apache.ignite.transactions.TransactionTimeoutException;
+
+import static org.apache.ignite.internal.util.typedef.X.hasCause;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ *
+ */
+public class TxDeadlockDetectionUnmasrhalErrorsTest extends GridCommonAbstractTest {
+    /** Nodes count. */
+    private static final int NODES_CNT = 2;
+
+    /** Client. */
+    private static boolean client;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(client);
+
+        if (isDebug()) {
+            TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+            discoSpi.failureDetectionTimeoutEnabled(false);
+
+            cfg.setDiscoverySpi(discoSpi);
+        }
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid(0);
+
+        client = true;
+
+        startGrid(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeadlockCacheObjectContext() throws Exception {
+        IgniteCache<Integer, Integer> cache0 = null;
+        IgniteCache<Integer, Integer> cache1 = null;
+        try {
+            cache0 = getCache(ignite(0), "cache0");
+            cache1 = getCache(ignite(0), "cache1");
+
+            IgniteCache<Integer, Integer> clientCache0 = grid(1).cache("cache0");
+
+            awaitPartitionMapExchange();
+
+            final CyclicBarrier barrier = new CyclicBarrier(2);
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            final AtomicInteger threadCnt = new AtomicInteger();
+
+            final AtomicBoolean deadlock = new AtomicBoolean();
+
+            IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+                @Override public void run() {
+                    int threadNum = threadCnt.getAndIncrement();
+
+                    Ignite ignite = ignite(0);
+
+                    IgniteCache<Integer, Integer> cache1 = ignite.cache("cache" + (threadNum == 0 ? 0 : 1));
+
+                    IgniteCache<Integer, Integer> cache2 = ignite.cache("cache" + (threadNum == 0 ? 1 : 0));
+
+                    try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 1000, 0)) {
+                        int key1 = threadNum == 0 ? 0 : 1;
+
+                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
+                            ", tx=" + tx + ", key=" + key1 + ", cache=" + cache1.getName() + ']');
+
+                        cache1.put(key1, 0);
+
+                        barrier.await();
+
+                        int key2 = threadNum == 0 ? 1 : 0;
+
+                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
+                            ", tx=" + tx + ", key=" + key2 + ", cache=" + cache2.getName() + ']');
+
+                        latch.countDown();
+
+                        cache2.put(key2, 1);
+
+                        tx.commit();
+
+                        log.info(">>> Commit done");
+                    }
+                    catch (Throwable e) {
+                        // At least one stack trace should contain TransactionDeadlockException.
+                        if (hasCause(e, TransactionTimeoutException.class) &&
+                            hasCause(e, TransactionDeadlockException.class)
+                            ) {
+                            if (deadlock.compareAndSet(false, true))
+                                U.error(log, "At least one stack trace should contain " +
+                                    TransactionDeadlockException.class.getSimpleName(), e);
+                        }
+                    }
+                }
+            }, 2, "tx-thread");
+
+            latch.await();
+
+            Ignite client = grid(1);
+
+            try (Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 500, 0)) {
+                clientCache0.put(0, 3);
+                clientCache0.put(1, 3);
+
+                tx.commit();
+
+                log.info(">>> Commit done");
+            }
+            catch (CacheException e) {
+                assertTrue(X.hasCause(e, TransactionTimeoutException.class));
+            }
+            catch (Throwable e) {
+                log.error("Unexpected exception occurred", e);
+
+                fail();
+            }
+
+            fut.get();
+
+            assertTrue(deadlock.get());
+
+            for (int i = 0; i < NODES_CNT ; i++) {
+                Ignite ignite = ignite(i);
+
+                IgniteTxManager txMgr = ((IgniteKernal)ignite).context().cache().context().tm();
+
+                Collection<IgniteInternalFuture<?>> futs = txMgr.deadlockDetectionFutures();
+
+                assertTrue(futs.isEmpty());
+            }
+
+            //assertNotNull(grid(1).context().cache().context().cacheContext(cacheId));
+        }
+        finally {
+            if (cache0 != null)
+                cache0.destroy();
+
+            if (cache1 != null)
+                cache1.destroy();
+        }
+    }
+
+
+
+    /**
+     * @param ignite Ignite.
+     * @param name Name.
+     */
+    private IgniteCache<Integer, Integer> getCache(Ignite ignite, String name) {
+        CacheConfiguration ccfg = defaultCacheConfiguration();
+
+        ccfg.setName(name);
+        ccfg.setCacheMode(CacheMode.PARTITIONED);
+        ccfg.setBackups(0);
+        ccfg.setNearConfiguration(null);
+
+        return ignite.getOrCreateCache(ccfg);
+    }
+
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java
new file mode 100644
index 0000000000000..f75750916728c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionDeadlockException;
+import org.apache.ignite.transactions.TransactionTimeoutException;
+
+import static org.apache.ignite.internal.util.typedef.X.hasCause;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ *
+ */
+public class TxDeadlockNpeClientTest extends GridCommonAbstractTest {
+    /** Nodes count. */
+    private static final int NODES_CNT = 2;
+
+    /** Client. */
+    private static boolean client;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(client);
+
+        if (isDebug()) {
+            TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+            discoSpi.failureDetectionTimeoutEnabled(false);
+
+            cfg.setDiscoverySpi(discoSpi);
+        }
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrid(0);
+
+        client = true;
+
+        startGrid(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeadlockCacheObjectContext() throws Exception {
+        IgniteCache<Integer, Integer> cache0 = null;
+        IgniteCache<Integer, Integer> cache1 = null;
+
+        try {
+            cache0 = getCache(ignite(0), "cache0");
+            cache1 = getCache(ignite(0), "cache1");
+
+            IgniteCache<Integer, Integer> clientCache0 = grid(1).cache("cache0");
+
+            awaitPartitionMapExchange();
+
+            final CyclicBarrier barrier = new CyclicBarrier(2);
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            final AtomicInteger threadCnt = new AtomicInteger();
+
+            final AtomicBoolean deadlock = new AtomicBoolean();
+
+            IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+                @Override public void run() {
+                    int threadNum = threadCnt.getAndIncrement();
+
+                    Ignite ignite = ignite(0);
+
+                    IgniteCache<Integer, Integer> cache1 = ignite.cache("cache" + (threadNum == 0 ? 0 : 1));
+
+                    IgniteCache<Integer, Integer> cache2 = ignite.cache("cache" + (threadNum == 0 ? 1 : 0));
+
+                    try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 1000, 0)) {
+                        int key1 = threadNum == 0 ? 0 : 1;
+
+                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
+                            ", tx=" + tx + ", key=" + key1 + ", cache=" + cache1.getName() + ']');
+
+                        cache1.put(key1, 0);
+
+                        barrier.await();
+
+                        int key2 = threadNum == 0 ? 1 : 0;
+
+                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
+                            ", tx=" + tx + ", key=" + key2 + ", cache=" + cache2.getName() + ']');
+
+                        latch.countDown();
+
+                        cache2.put(key2, 1);
+
+                        tx.commit();
+
+                        log.info(">>> Commit done");
+                    }
+                    catch (Throwable e) {
+                        // At least one stack trace should contain TransactionDeadlockException.
+                        if (hasCause(e, TransactionTimeoutException.class) &&
+                            hasCause(e, TransactionDeadlockException.class)
+                            ) {
+                            if (deadlock.compareAndSet(false, true))
+                                U.error(log, "At least one stack trace should contain " +
+                                    TransactionDeadlockException.class.getSimpleName(), e);
+                        }
+                    }
+                }
+            }, 2, "tx-thread");
+
+            latch.await();
+
+            Ignite client = grid(1);
+
+            try (Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 500, 0)) {
+                clientCache0.put(0, 3);
+                clientCache0.put(1, 3);
+
+                tx.commit();
+
+                log.info(">>> Commit done");
+            }
+            catch (CacheException e) {
+                assertTrue(X.hasCause(e, TransactionTimeoutException.class));
+            }
+            catch (Throwable e) {
+                fail();
+            }
+
+            fut.get();
+
+            assertTrue(deadlock.get());
+
+            for (int i = 0; i < NODES_CNT ; i++) {
+                Ignite ignite = ignite(i);
+
+                IgniteTxManager txMgr = ((IgniteKernal)ignite).context().cache().context().tm();
+
+                Collection<IgniteInternalFuture<?>> futs = txMgr.deadlockDetectionFutures();
+
+                assertTrue(futs.isEmpty());
+            }
+        }
+        finally {
+            if (cache0 != null)
+                cache0.destroy();
+
+            if (cache1 != null)
+                cache1.destroy();
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @param name Name.
+     */
+    private IgniteCache<Integer, Integer> getCache(Ignite ignite, String name) {
+        CacheConfiguration ccfg = defaultCacheConfiguration();
+
+        ccfg.setName(name);
+        ccfg.setCacheMode(CacheMode.PARTITIONED);
+        ccfg.setBackups(0);
+        ccfg.setNearConfiguration(null);
+
+        return ignite.getOrCreateCache(ccfg);
+    }
+
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java
index 5a1b1ad37168a..337d4d46a06d0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java
@@ -19,8 +19,10 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.transactions.DepthFirstSearchTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionMessageMarshallingTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionNoHangsTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionUnmasrhalErrorsTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticDeadlockDetectionCrossCacheTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticDeadlockDetectionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxPessimisticDeadlockDetectionCrossCacheTest;
@@ -44,6 +46,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TxPessimisticDeadlockDetectionCrossCacheTest.class);
         suite.addTestSuite(TxDeadlockDetectionTest.class);
         suite.addTestSuite(TxDeadlockDetectionNoHangsTest.class);
+        suite.addTestSuite(TxDeadlockDetectionUnmasrhalErrorsTest.class);
+        suite.addTestSuite(TxDeadlockDetectionMessageMarshallingTest.class);
 
         return suite;
     }

From a1fec4b0b3ddcc208154114e0bf8f202eb597fc9 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 25 Apr 2017 09:14:07 +0300
Subject: [PATCH 141/516] Fix index initialization.

---
 .../ignite/internal/processors/query/h2/IgniteH2Indexing.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 407819746d4b5..85fb4989658d3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -2899,7 +2899,7 @@ private SpatialIndex createH2SpatialIndex(
         private Index createTreeIndex(String idxName, GridH2Table tbl, boolean pk, List<IndexColumn> columns) {
             final int segments = tbl.rowDescriptor().configuration().getQueryParallelism();
 
-            return new GridH2TreeIndex(idxName, tbl, pk, columns, segments);
+            return new GridH2TreeIndex(idxName, tbl, pk, columns, segments > 0 ? segments : 1);
         }
     }
 

From d94172e7bb7ff4ebab422997e5c70f8cd03c0ae4 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 26 Apr 2017 15:16:46 +0300
Subject: [PATCH 142/516] Redundant test removed

---
 .../transactions/TxDeadlockNpeClientTest.java | 220 ------------------
 1 file changed, 220 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java
deleted file mode 100644
index f75750916728c..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockNpeClientTest.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.transactions;
-
-import java.util.Collection;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.cache.CacheException;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.transactions.Transaction;
-import org.apache.ignite.transactions.TransactionDeadlockException;
-import org.apache.ignite.transactions.TransactionTimeoutException;
-
-import static org.apache.ignite.internal.util.typedef.X.hasCause;
-import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
-import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
-import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
-
-/**
- *
- */
-public class TxDeadlockNpeClientTest extends GridCommonAbstractTest {
-    /** Nodes count. */
-    private static final int NODES_CNT = 2;
-
-    /** Client. */
-    private static boolean client;
-
-    /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        cfg.setClientMode(client);
-
-        if (isDebug()) {
-            TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
-            discoSpi.failureDetectionTimeoutEnabled(false);
-
-            cfg.setDiscoverySpi(discoSpi);
-        }
-
-        return cfg;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        super.beforeTestsStarted();
-
-        startGrid(0);
-
-        client = true;
-
-        startGrid(1);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        super.afterTestsStopped();
-
-        stopAllGrids();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testDeadlockCacheObjectContext() throws Exception {
-        IgniteCache<Integer, Integer> cache0 = null;
-        IgniteCache<Integer, Integer> cache1 = null;
-
-        try {
-            cache0 = getCache(ignite(0), "cache0");
-            cache1 = getCache(ignite(0), "cache1");
-
-            IgniteCache<Integer, Integer> clientCache0 = grid(1).cache("cache0");
-
-            awaitPartitionMapExchange();
-
-            final CyclicBarrier barrier = new CyclicBarrier(2);
-
-            final CountDownLatch latch = new CountDownLatch(1);
-
-            final AtomicInteger threadCnt = new AtomicInteger();
-
-            final AtomicBoolean deadlock = new AtomicBoolean();
-
-            IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
-                @Override public void run() {
-                    int threadNum = threadCnt.getAndIncrement();
-
-                    Ignite ignite = ignite(0);
-
-                    IgniteCache<Integer, Integer> cache1 = ignite.cache("cache" + (threadNum == 0 ? 0 : 1));
-
-                    IgniteCache<Integer, Integer> cache2 = ignite.cache("cache" + (threadNum == 0 ? 1 : 0));
-
-                    try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 1000, 0)) {
-                        int key1 = threadNum == 0 ? 0 : 1;
-
-                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
-                            ", tx=" + tx + ", key=" + key1 + ", cache=" + cache1.getName() + ']');
-
-                        cache1.put(key1, 0);
-
-                        barrier.await();
-
-                        int key2 = threadNum == 0 ? 1 : 0;
-
-                        log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
-                            ", tx=" + tx + ", key=" + key2 + ", cache=" + cache2.getName() + ']');
-
-                        latch.countDown();
-
-                        cache2.put(key2, 1);
-
-                        tx.commit();
-
-                        log.info(">>> Commit done");
-                    }
-                    catch (Throwable e) {
-                        // At least one stack trace should contain TransactionDeadlockException.
-                        if (hasCause(e, TransactionTimeoutException.class) &&
-                            hasCause(e, TransactionDeadlockException.class)
-                            ) {
-                            if (deadlock.compareAndSet(false, true))
-                                U.error(log, "At least one stack trace should contain " +
-                                    TransactionDeadlockException.class.getSimpleName(), e);
-                        }
-                    }
-                }
-            }, 2, "tx-thread");
-
-            latch.await();
-
-            Ignite client = grid(1);
-
-            try (Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 500, 0)) {
-                clientCache0.put(0, 3);
-                clientCache0.put(1, 3);
-
-                tx.commit();
-
-                log.info(">>> Commit done");
-            }
-            catch (CacheException e) {
-                assertTrue(X.hasCause(e, TransactionTimeoutException.class));
-            }
-            catch (Throwable e) {
-                fail();
-            }
-
-            fut.get();
-
-            assertTrue(deadlock.get());
-
-            for (int i = 0; i < NODES_CNT ; i++) {
-                Ignite ignite = ignite(i);
-
-                IgniteTxManager txMgr = ((IgniteKernal)ignite).context().cache().context().tm();
-
-                Collection<IgniteInternalFuture<?>> futs = txMgr.deadlockDetectionFutures();
-
-                assertTrue(futs.isEmpty());
-            }
-        }
-        finally {
-            if (cache0 != null)
-                cache0.destroy();
-
-            if (cache1 != null)
-                cache1.destroy();
-        }
-    }
-
-    /**
-     * @param ignite Ignite.
-     * @param name Name.
-     */
-    private IgniteCache<Integer, Integer> getCache(Ignite ignite, String name) {
-        CacheConfiguration ccfg = defaultCacheConfiguration();
-
-        ccfg.setName(name);
-        ccfg.setCacheMode(CacheMode.PARTITIONED);
-        ccfg.setBackups(0);
-        ccfg.setNearConfiguration(null);
-
-        return ignite.getOrCreateCache(ccfg);
-    }
-
-
-}

From c64dc2426a70896116dbdffbf90faea5e52b991c Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 27 Apr 2017 16:35:25 +0300
Subject: [PATCH 143/516] Revert IGNITE-4863 partially. Warn user if default
 log level was changed. As default log level value (DEBUG) can cause
 performance issues.

---
 .../ignite/logger/log4j/Log4JLogger.java      | 51 +++++++++----------
 .../log4j/GridLog4jInitializationTest.java    |  4 +-
 2 files changed, 27 insertions(+), 28 deletions(-)

diff --git a/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java b/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
index f6ed83025af55..a49e24716c399 100644
--- a/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
+++ b/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
@@ -56,20 +56,20 @@
  * <p>
  * Here is a typical example of configuring log4j logger in Ignite configuration file:
  * <pre name="code" class="xml">
- *      &lt;property name="gridLogger"&gt;
- *          &lt;bean class="org.apache.ignite.logger.log4j.Log4JLogger"&gt;
- *              &lt;constructor-arg type="java.lang.String" value="config/ignite-log4j.xml"/&gt;
- *          &lt;/bean>
- *      &lt;/property&gt;
+ * &lt;property name="gridLogger"&gt;
+ * &lt;bean class="org.apache.ignite.logger.log4j.Log4JLogger"&gt;
+ * &lt;constructor-arg type="java.lang.String" value="config/ignite-log4j.xml"/&gt;
+ * &lt;/bean>
+ * &lt;/property&gt;
  * </pre>
  * and from your code:
  * <pre name="code" class="java">
- *      IgniteConfiguration cfg = new IgniteConfiguration();
- *      ...
- *      URL xml = U.resolveIgniteUrl("config/custom-log4j.xml");
- *      IgniteLogger log = new Log4JLogger(xml);
- *      ...
- *      cfg.setGridLogger(log);
+ * IgniteConfiguration cfg = new IgniteConfiguration();
+ * ...
+ * URL xml = U.resolveIgniteUrl("config/custom-log4j.xml");
+ * IgniteLogger log = new Log4JLogger(xml);
+ * ...
+ * cfg.setGridLogger(log);
  * </pre>
  *
  * Please take a look at <a target=_new href="http://logging.apache.org/log4j/1.2/index.html">Apache Log4j 1.2</a>
@@ -352,21 +352,20 @@ private void addConsoleAppenderIfNeeded(@Nullable Level logLevel,
                     if (errAppender.getThreshold() == Level.ERROR)
                         errAppender.setThreshold(Level.WARN);
                 }
-                else {
-                    // No error console appender => create console appender with.
-                    final AppenderSkeleton consoleAppender = createConsoleAppender(Level.OFF);
-
-                    consoleAppender.setThreshold(Level.INFO);
-
-                    rootCategory.addAppender(consoleAppender);
-                }
+                else
+                    // No error console appender => create console appender with no level limit.
+                    rootCategory.addAppender(createConsoleAppender(Level.OFF));
 
-                // Won't raise LogLevel if there is other loggers configured. As LogLevel can be inherited.
-                if (logLevel != null && !logLevel.isGreaterOrEqual(impl.getEffectiveLevel())) {
+                if (logLevel != null) {
                     impl.setLevel(logLevel);
 
-                    impl.warn("RootLogger logging level has been dropped by Apache Ignite.\n"+
-                    "Set lower log level or configure ConsoleAppender manually or disable ConsoleAppender automatic creation.");
+                    // Warn if LogLevel is changed and there is other logger configured that can inherits LogLevel.
+                    if (!impl.getLevel().equals(logLevel) && hasOtherLoggers())
+                        impl.warn("RootLogger logging level has been changed by Apache Ignite to " +
+                            logLevel.toString() + " level.\n" +
+                            "Please, configure ConsoleAppender manually or disable ConsoleAppender automatic creation.");
+                    else
+                        impl.info("Apache Ignite set default logging level to " + logLevel.toString());
                 }
             }
             else if (!isConfigured() && !hasOtherLoggers()) {
@@ -403,11 +402,11 @@ private boolean hasOtherLoggers() {
      * @param maxLevel Max logging level.
      * @return New console appender.
      */
-    private AppenderSkeleton createConsoleAppender(Level maxLevel) {
+    private Appender createConsoleAppender(Level maxLevel) {
         String fmt = "[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n";
 
         // Configure output that should go to System.out
-        AppenderSkeleton app = new ConsoleAppender(new PatternLayout(fmt), ConsoleAppender.SYSTEM_OUT);
+        Appender app = new ConsoleAppender(new PatternLayout(fmt), ConsoleAppender.SYSTEM_OUT);
 
         LevelRangeFilter lvlFilter = new LevelRangeFilter();
 
@@ -569,7 +568,7 @@ public static Collection<String> logFiles() {
     /**
      * For test purposes only.
      */
-    static void reset(){
+    static void reset() {
         inited = false;
 
         quiet0 = false;
diff --git a/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializationTest.java b/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializationTest.java
index 2a98490744db5..1f35c8ec55090 100644
--- a/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializationTest.java
+++ b/modules/log4j/src/test/java/org/apache/ignite/logger/log4j/GridLog4jInitializationTest.java
@@ -133,7 +133,7 @@ public void testAutoAddConsoleAppender2() {
         if (VERBOSE)
             printLoggerResults(log);
 
-        assertEquals(Level.DEBUG, logger.getEffectiveLevel()); // LogLevel should not change.
+        assertEquals(Level.INFO, logger.getEffectiveLevel());
     }
 
     /** */
@@ -171,7 +171,7 @@ public void testAutoAddConsoleAppenderWithOtherLoggerConfigured() {
         if (VERBOSE)
             printLoggerResults(log);
 
-        assertEquals(Level.DEBUG, logger.getEffectiveLevel()); // LogLevel should not be raised.
+        assertEquals(Level.INFO, logger.getEffectiveLevel());
     }
 
     /** */

From f9ecacc625b458539775e6550bd9b7613ed38f21 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 28 Apr 2017 11:46:23 +0300
Subject: [PATCH 144/516] IGNITE-5077 - Support service security permissions

backport from master
(cherry picked from commit 6236b5f)
---
 .../processors/security/SecurityContext.java  |  9 ++++++
 .../service/GridServiceProcessor.java         | 11 +++++++
 .../security/SecurityBasicPermissionSet.java  | 17 ++++++++++
 .../plugin/security/SecurityPermission.java   | 13 ++++++--
 .../security/SecurityPermissionSet.java       |  8 +++++
 .../SecurityPermissionSetBuilder.java         | 19 +++++++++++
 .../SecurityPermissionSetBuilderTest.java     | 32 +++++++++++++++----
 .../junits/spi/GridSpiAbstractTest.java       |  5 +++
 8 files changed, 106 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContext.java
index ef46713a72f63..bf5894ef3f48c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContext.java
@@ -47,6 +47,15 @@ public interface SecurityContext {
      */
     public boolean cacheOperationAllowed(String cacheName, SecurityPermission perm);
 
+    /**
+     * Checks whether service operation is allowed.
+     *
+     * @param srvcName Service name.
+     * @param perm Permission to check.
+     * @return {@code True} if task operation is allowed.
+     */
+    public boolean serviceOperationAllowed(String srvcName, SecurityPermission perm);
+
     /**
      * Checks whether system-wide permission is allowed (excluding Visor task operations).
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 2a363e21e8ef5..d7b9abc7fc1bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -92,6 +92,7 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
@@ -496,6 +497,8 @@ public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
 
         validate(cfg);
 
+        ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
+
         if (!state.srvcCompatibility) {
             Marshaller marsh = ctx.config().getMarshaller();
 
@@ -632,6 +635,8 @@ private ServicesCompatibilityState markCompatibilityStateAsUsed() {
      * @return Future.
      */
     public IgniteInternalFuture<?> cancel(String name) {
+        ctx.security().authorize(name, SecurityPermission.SERVICE_CANCEL, null);
+
         while (true) {
             try {
                 GridFutureAdapter<?> fut = new GridFutureAdapter<>();
@@ -780,6 +785,8 @@ public Collection<ServiceDescriptor> serviceDescriptors() {
      */
     @SuppressWarnings("unchecked")
     public <T> T service(String name) {
+        ctx.security().authorize(name, SecurityPermission.SERVICE_INVOKE, null);
+
         Collection<ServiceContextImpl> ctxs;
 
         synchronized (locSvcs) {
@@ -844,6 +851,8 @@ public ServiceContextImpl serviceContext(String name) {
     @SuppressWarnings("unchecked")
     public <T> T serviceProxy(ClusterGroup prj, String name, Class<? super T> svcItf, boolean sticky, long timeout)
         throws IgniteException {
+        ctx.security().authorize(name, SecurityPermission.SERVICE_INVOKE, null);
+
         if (hasLocalNode(prj)) {
             ServiceContextImpl ctx = serviceContext(name);
 
@@ -883,6 +892,8 @@ private boolean hasLocalNode(ClusterGroup prj) {
      */
     @SuppressWarnings("unchecked")
     public <T> Collection<T> services(String name) {
+        ctx.security().authorize(name, SecurityPermission.SERVICE_INVOKE, null);
+
         Collection<ServiceContextImpl> ctxs;
 
         synchronized (locSvcs) {
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
index 5b50c56321377..7521dff84df4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
@@ -38,6 +38,9 @@ public class SecurityBasicPermissionSet implements SecurityPermissionSet {
     /** Task permissions. */
     private Map<String, Collection<SecurityPermission>> taskPerms = new HashMap<>();
 
+    /** Service permissions. */
+    private Map<String, Collection<SecurityPermission>> srvcPerms = new HashMap<>();
+
     /** System permissions. */
     private Collection<SecurityPermission> sysPerms = new ArrayList<>();
 
@@ -62,6 +65,15 @@ public void setTaskPermissions(Map<String, Collection<SecurityPermission>> taskP
         this.taskPerms = taskPerms;
     }
 
+    /**
+     * Setter for set service permission map.
+     *
+     * @param srvcPerms Service permissions.
+     */
+    public void setServicePermissions(Map<String, Collection<SecurityPermission>> srvcPerms) {
+        this.srvcPerms = srvcPerms;
+    }
+
     /**
      * Setter for set collection  system permission.
      *
@@ -90,6 +102,11 @@ public void setDefaultAllowAll(boolean dfltAllowAll) {
         return taskPerms;
     }
 
+    /** {@inheritDoc} */
+    @Override public Map<String, Collection<SecurityPermission>> servicePermissions() {
+        return srvcPerms;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public Collection<SecurityPermission> systemPermissions() {
         return sysPerms;
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
index 9f63c1ed1d2a2..54361614e62a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
@@ -21,7 +21,7 @@
 
 /**
  * Supported security permissions within grid. Permissions
- * are specified on per-cache or per-task level.
+ * are specified on per-cache, per-task or per-service level.
  */
 public enum SecurityPermission {
     /** Cache {@code read} permission. */
@@ -55,7 +55,16 @@ public enum SecurityPermission {
     ADMIN_CACHE,
 
     /** Visor admin operations permissions. */
-    ADMIN_OPS;
+    ADMIN_OPS,
+
+    /** Service deploy permission. */
+    SERVICE_DEPLOY,
+
+    /** Service cancel permission. */
+    SERVICE_CANCEL,
+
+    /** Service invoke permission. */
+    SERVICE_INVOKE;
 
     /** Enumerated values. */
     private static final SecurityPermission[] VALS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSet.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSet.java
index 99615019c10d2..5e07e421eaf78 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSet.java
@@ -58,6 +58,14 @@ public interface SecurityPermissionSet extends Serializable, LessNamingBean {
      */
     public Map<String, Collection<SecurityPermission>> cachePermissions();
 
+    /**
+     * Map of service names to service permissions. Wildcards are allowed at the
+     * end of service names.
+     *
+     * @return Map of service names to service permissions.
+     */
+    public Map<String, Collection<SecurityPermission>> servicePermissions();
+
     /**
      * Collection of system-wide permissions (events enable/disable, Visor task execution).
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
index 61ad77ce08f5e..cf38c0f0e7e90 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
@@ -57,6 +57,9 @@ public class SecurityPermissionSetBuilder {
     /** Task permissions.*/
     private Map<String, Collection<SecurityPermission>> taskPerms = new HashMap<>();
 
+    /** Service permissions.*/
+    private Map<String, Collection<SecurityPermission>> srvcPerms = new HashMap<>();
+
     /** System permissions.*/
     private List<SecurityPermission> sysPerms = new ArrayList<>();
 
@@ -99,6 +102,21 @@ public SecurityPermissionSetBuilder appendTaskPermissions(String name, SecurityP
         return this;
     }
 
+    /**
+     * Append permission set form {@link org.apache.ignite.IgniteServices service} with {@code name}.
+     *
+     * @param name  String for map some service to permission set.
+     * @param perms Permissions.
+     * @return SecurityPermissionSetBuilder refer to same permission builder.
+     */
+    public SecurityPermissionSetBuilder appendServicePermissions(String name, SecurityPermission... perms) {
+        validate(toCollection("SERVICE_"), perms);
+
+        append(srvcPerms, name, toCollection(perms));
+
+        return this;
+    }
+
     /**
      * Append permission set form {@link org.apache.ignite.IgniteCache cache} with {@code name}.
      *
@@ -215,6 +233,7 @@ public SecurityPermissionSet build() {
         permSet.setDefaultAllowAll(dfltAllowAll);
         permSet.setCachePermissions(unmodifiableMap(cachePerms));
         permSet.setTaskPermissions(unmodifiableMap(taskPerms));
+        permSet.setServicePermissions(unmodifiableMap(srvcPerms));
         permSet.setSystemPermissions(unmodifiableList(sysPerms));
 
         return permSet;
diff --git a/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java b/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
index f63f9a7da0a01..5443cfd77fd5c 100644
--- a/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
@@ -28,6 +28,8 @@
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_PUT;
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_READ;
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_REMOVE;
+import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_DEPLOY;
+import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_INVOKE;
 import static org.apache.ignite.plugin.security.SecurityPermission.TASK_CANCEL;
 import static org.apache.ignite.plugin.security.SecurityPermission.TASK_EXECUTE;
 import static org.apache.ignite.plugin.security.SecurityPermission.EVENTS_ENABLE;
@@ -41,6 +43,7 @@ public class SecurityPermissionSetBuilderTest extends GridCommonAbstractTest {
     /**
      *
      */
+    @SuppressWarnings({"ThrowableNotThrown", "ArraysAsListWithZeroOrOneArgument"})
     public void testPermissionBuilder() {
         SecurityBasicPermissionSet exp = new SecurityBasicPermissionSet();
 
@@ -56,13 +59,18 @@ public void testPermissionBuilder() {
 
         exp.setTaskPermissions(permTask);
 
+        Map<String, Collection<SecurityPermission>> permSrvc = new HashMap<>();
+        permSrvc.put("service1", Arrays.asList(SERVICE_DEPLOY));
+        permSrvc.put("service2", Arrays.asList(SERVICE_INVOKE));
+
+        exp.setServicePermissions(permSrvc);
+
         exp.setSystemPermissions(Arrays.asList(ADMIN_VIEW, EVENTS_ENABLE));
 
         final SecurityPermissionSetBuilder permsBuilder = new SecurityPermissionSetBuilder();
 
         assertThrows(log, new Callable<Object>() {
-                    @Override
-                    public Object call() throws Exception {
+                    @Override public Object call() throws Exception {
                         permsBuilder.appendCachePermissions("cache", ADMIN_VIEW);
                         return null;
                     }
@@ -71,8 +79,7 @@ public Object call() throws Exception {
         );
 
         assertThrows(log, new Callable<Object>() {
-                    @Override
-                    public Object call() throws Exception {
+                    @Override public Object call() throws Exception {
                         permsBuilder.appendTaskPermissions("task", CACHE_READ);
                         return null;
                     }
@@ -81,8 +88,7 @@ public Object call() throws Exception {
         );
 
         assertThrows(log, new Callable<Object>() {
-                    @Override
-                    public Object call() throws Exception {
+                    @Override public Object call() throws Exception {
                         permsBuilder.appendSystemPermissions(TASK_EXECUTE, CACHE_PUT);
                         return null;
                     }
@@ -90,6 +96,15 @@ public Object call() throws Exception {
                 "you can assign permission only start with [EVENTS_, ADMIN_], but you try TASK_EXECUTE"
         );
 
+        assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    permsBuilder.appendSystemPermissions(SERVICE_INVOKE, CACHE_REMOVE);
+                    return null;
+                }
+            }, IgniteException.class,
+            "you can assign permission only start with [EVENTS_, ADMIN_], but you try SERVICE_INVOKE"
+        );
+
         permsBuilder.appendCachePermissions(
                 "cache1", CACHE_PUT, CACHE_REMOVE
         ).appendCachePermissions(
@@ -98,12 +113,17 @@ public Object call() throws Exception {
                 "task1", TASK_CANCEL
         ).appendTaskPermissions(
                 "task2", TASK_EXECUTE
+        ).appendServicePermissions(
+            "service1", SERVICE_DEPLOY
+        ).appendServicePermissions(
+            "service2", SERVICE_INVOKE
         ).appendSystemPermissions(ADMIN_VIEW, EVENTS_ENABLE);
 
         SecurityPermissionSet actual = permsBuilder.build();
 
         assertEquals(exp.cachePermissions(), actual.cachePermissions());
         assertEquals(exp.taskPermissions(), actual.taskPermissions());
+        assertEquals(exp.servicePermissions(), actual.servicePermissions());
         assertEquals(exp.systemPermissions(), actual.systemPermissions());
         assertEquals(exp.defaultAllowAll(), actual.defaultAllowAll());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java
index 20b3cf2fdb354..0aeff3c08b78a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java
@@ -717,6 +717,11 @@ private static class SecurityPermissionSetImpl implements SecurityPermissionSet
             return Collections.emptyMap();
         }
 
+        /** {@inheritDoc} */
+        @Override public Map<String, Collection<SecurityPermission>> servicePermissions() {
+            return Collections.emptyMap();
+        }
+
         /** {@inheritDoc} */
         @Nullable @Override public Collection<SecurityPermission> systemPermissions() {
             return null;

From 91c899b909383c78b78b9bf0c8f233b8c75ef29e Mon Sep 17 00:00:00 2001
From: Valentin Kulichenko <valentin.kulichenko@gmail.com>
Date: Fri, 28 Apr 2017 14:48:57 +0200
Subject: [PATCH 145/516] IGNITE-5081 - Removed redundant duplication of
 permissions in SecurityPermissionSetBuilder

---
 .../SecurityPermissionSetBuilder.java         | 17 ++---
 .../SecurityPermissionSetBuilderTest.java     | 63 +++++++++++--------
 2 files changed, 46 insertions(+), 34 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
index cf38c0f0e7e90..abac541ffdb68 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java
@@ -17,16 +17,17 @@
 
 package org.apache.ignite.plugin.security;
 
-import java.util.Map;
-import java.util.List;
-import java.util.HashMap;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
-import static java.util.Collections.unmodifiableList;
 import static java.util.Collections.unmodifiableMap;
+import static java.util.Collections.unmodifiableSet;
 
 /**
  * Provides a convenient way to create a permission set.
@@ -61,7 +62,7 @@ public class SecurityPermissionSetBuilder {
     private Map<String, Collection<SecurityPermission>> srvcPerms = new HashMap<>();
 
     /** System permissions.*/
-    private List<SecurityPermission> sysPerms = new ArrayList<>();
+    private Set<SecurityPermission> sysPerms = new HashSet<>();
 
     /** Default allow all.*/
     private boolean dfltAllowAll;
@@ -193,7 +194,7 @@ private void validate(Collection<String> ptrns, SecurityPermission perm) {
     private final <T> Collection<T> toCollection(T... perms) {
         assert perms != null;
 
-        Collection<T> col = new ArrayList<>(perms.length);
+        Collection<T> col = U.newHashSet(perms.length);
 
         Collections.addAll(col, perms);
 
@@ -234,7 +235,7 @@ public SecurityPermissionSet build() {
         permSet.setCachePermissions(unmodifiableMap(cachePerms));
         permSet.setTaskPermissions(unmodifiableMap(taskPerms));
         permSet.setServicePermissions(unmodifiableMap(srvcPerms));
-        permSet.setSystemPermissions(unmodifiableList(sysPerms));
+        permSet.setSystemPermissions(unmodifiableSet(sysPerms));
 
         return permSet;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java b/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
index 5443cfd77fd5c..0ac7bc739f179 100644
--- a/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java
@@ -17,23 +17,24 @@
 
 package org.apache.ignite.plugin.security;
 
-import java.util.Map;
-import java.util.Arrays;
-import java.util.HashMap;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.Callable;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_VIEW;
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_PUT;
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_READ;
 import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_REMOVE;
 import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_DEPLOY;
 import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_INVOKE;
+import static org.apache.ignite.plugin.security.SecurityPermission.EVENTS_ENABLE;
 import static org.apache.ignite.plugin.security.SecurityPermission.TASK_CANCEL;
 import static org.apache.ignite.plugin.security.SecurityPermission.TASK_EXECUTE;
-import static org.apache.ignite.plugin.security.SecurityPermission.EVENTS_ENABLE;
-import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_VIEW;
 import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
 
 /**
@@ -41,31 +42,30 @@
  */
 public class SecurityPermissionSetBuilderTest extends GridCommonAbstractTest {
     /**
-     *
      */
     @SuppressWarnings({"ThrowableNotThrown", "ArraysAsListWithZeroOrOneArgument"})
     public void testPermissionBuilder() {
         SecurityBasicPermissionSet exp = new SecurityBasicPermissionSet();
 
         Map<String, Collection<SecurityPermission>> permCache = new HashMap<>();
-        permCache.put("cache1", Arrays.asList(CACHE_PUT, CACHE_REMOVE));
-        permCache.put("cache2", Arrays.asList(CACHE_READ));
+        permCache.put("cache1", permissions(CACHE_PUT, CACHE_REMOVE));
+        permCache.put("cache2", permissions(CACHE_READ));
 
         exp.setCachePermissions(permCache);
 
         Map<String, Collection<SecurityPermission>> permTask = new HashMap<>();
-        permTask.put("task1", Arrays.asList(TASK_CANCEL));
-        permTask.put("task2", Arrays.asList(TASK_EXECUTE));
+        permTask.put("task1", permissions(TASK_CANCEL));
+        permTask.put("task2", permissions(TASK_EXECUTE));
 
         exp.setTaskPermissions(permTask);
 
         Map<String, Collection<SecurityPermission>> permSrvc = new HashMap<>();
-        permSrvc.put("service1", Arrays.asList(SERVICE_DEPLOY));
-        permSrvc.put("service2", Arrays.asList(SERVICE_INVOKE));
+        permSrvc.put("service1", permissions(SERVICE_DEPLOY));
+        permSrvc.put("service2", permissions(SERVICE_INVOKE));
 
         exp.setServicePermissions(permSrvc);
 
-        exp.setSystemPermissions(Arrays.asList(ADMIN_VIEW, EVENTS_ENABLE));
+        exp.setSystemPermissions(permissions(ADMIN_VIEW, EVENTS_ENABLE));
 
         final SecurityPermissionSetBuilder permsBuilder = new SecurityPermissionSetBuilder();
 
@@ -105,19 +105,18 @@ public void testPermissionBuilder() {
             "you can assign permission only start with [EVENTS_, ADMIN_], but you try SERVICE_INVOKE"
         );
 
-        permsBuilder.appendCachePermissions(
-                "cache1", CACHE_PUT, CACHE_REMOVE
-        ).appendCachePermissions(
-                "cache2", CACHE_READ
-        ).appendTaskPermissions(
-                "task1", TASK_CANCEL
-        ).appendTaskPermissions(
-                "task2", TASK_EXECUTE
-        ).appendServicePermissions(
-            "service1", SERVICE_DEPLOY
-        ).appendServicePermissions(
-            "service2", SERVICE_INVOKE
-        ).appendSystemPermissions(ADMIN_VIEW, EVENTS_ENABLE);
+        permsBuilder
+            .appendCachePermissions("cache1", CACHE_PUT)
+            .appendCachePermissions("cache1", CACHE_PUT, CACHE_REMOVE)
+            .appendCachePermissions("cache2", CACHE_READ)
+            .appendTaskPermissions("task1", TASK_CANCEL)
+            .appendTaskPermissions("task2", TASK_EXECUTE)
+            .appendTaskPermissions("task2", TASK_EXECUTE)
+            .appendServicePermissions("service1", SERVICE_DEPLOY)
+            .appendServicePermissions("service2", SERVICE_INVOKE)
+            .appendServicePermissions("service2", SERVICE_INVOKE)
+            .appendSystemPermissions(ADMIN_VIEW)
+            .appendSystemPermissions(ADMIN_VIEW, EVENTS_ENABLE);
 
         SecurityPermissionSet actual = permsBuilder.build();
 
@@ -127,4 +126,16 @@ public void testPermissionBuilder() {
         assertEquals(exp.systemPermissions(), actual.systemPermissions());
         assertEquals(exp.defaultAllowAll(), actual.defaultAllowAll());
     }
+
+    /**
+     * @param perms Permissions.
+     * @return Collection.
+     */
+    private static Collection<SecurityPermission> permissions(SecurityPermission... perms) {
+        Collection<SecurityPermission> col = U.newHashSet(perms.length);
+
+        Collections.addAll(col, perms);
+
+        return col;
+    }
 }

From b48a26b9b1e97fb8eb52c2a2f36005770922ac3d Mon Sep 17 00:00:00 2001
From: Valentin Kulichenko <valentin.kulichenko@gmail.com>
Date: Fri, 28 Apr 2017 14:53:33 +0200
Subject: [PATCH 146/516] IGNITE-5080 - Fixes in SecurityBasicPermissionSet

---
 .../security/SecurityBasicPermissionSet.java  | 89 ++++++++++++++-----
 1 file changed, 65 insertions(+), 24 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
index 7521dff84df4f..44166d936c9aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
@@ -17,10 +17,12 @@
 
 package org.apache.ignite.plugin.security;
 
-import java.util.Map;
-import java.util.HashMap;
-import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
 
@@ -33,16 +35,20 @@ public class SecurityBasicPermissionSet implements SecurityPermissionSet {
     private static final long serialVersionUID = 0L;
 
     /** Cache permissions. */
-    private Map<String, Collection<SecurityPermission>> cachePerms = new HashMap<>();
+    @GridToStringInclude
+    private Map<String, Collection<SecurityPermission>> cachePermissions = new HashMap<>();
 
     /** Task permissions. */
-    private Map<String, Collection<SecurityPermission>> taskPerms = new HashMap<>();
+    @GridToStringInclude
+    private Map<String, Collection<SecurityPermission>> taskPermissions = new HashMap<>();
 
     /** Service permissions. */
-    private Map<String, Collection<SecurityPermission>> srvcPerms = new HashMap<>();
+    @GridToStringInclude
+    private Map<String, Collection<SecurityPermission>> servicePermissions = new HashMap<>();
 
     /** System permissions. */
-    private Collection<SecurityPermission> sysPerms = new ArrayList<>();
+    @GridToStringInclude
+    private Collection<SecurityPermission> systemPermissions;
 
     /** Default allow all. */
     private boolean dfltAllowAll;
@@ -50,37 +56,43 @@ public class SecurityBasicPermissionSet implements SecurityPermissionSet {
     /**
      * Setter for set cache permission map.
      *
-     * @param cachePerms Cache permissions.
+     * @param cachePermissions Cache permissions.
      */
-    public void setCachePermissions(Map<String, Collection<SecurityPermission>> cachePerms) {
-        this.cachePerms = cachePerms;
+    public void setCachePermissions(Map<String, Collection<SecurityPermission>> cachePermissions) {
+        A.notNull(cachePermissions, "cachePermissions");
+
+        this.cachePermissions = cachePermissions;
     }
 
     /**
      * Setter for set task permission map.
      *
-     * @param taskPerms Task permissions.
+     * @param taskPermissions Task permissions.
      */
-    public void setTaskPermissions(Map<String, Collection<SecurityPermission>> taskPerms) {
-        this.taskPerms = taskPerms;
+    public void setTaskPermissions(Map<String, Collection<SecurityPermission>> taskPermissions) {
+        A.notNull(taskPermissions, "taskPermissions");
+
+        this.taskPermissions = taskPermissions;
     }
 
     /**
      * Setter for set service permission map.
      *
-     * @param srvcPerms Service permissions.
+     * @param servicePermissions Service permissions.
      */
-    public void setServicePermissions(Map<String, Collection<SecurityPermission>> srvcPerms) {
-        this.srvcPerms = srvcPerms;
+    public void setServicePermissions(Map<String, Collection<SecurityPermission>> servicePermissions) {
+        A.notNull(taskPermissions, "servicePermissions");
+
+        this.servicePermissions = servicePermissions;
     }
 
     /**
-     * Setter for set collection  system permission.
+     * Setter for set collection system permission.
      *
-     * @param sysPerms System permissions.
+     * @param systemPermissions System permissions.
      */
-    public void setSystemPermissions(Collection<SecurityPermission> sysPerms) {
-        this.sysPerms = sysPerms;
+    public void setSystemPermissions(Collection<SecurityPermission> systemPermissions) {
+        this.systemPermissions = systemPermissions;
     }
 
     /**
@@ -94,22 +106,22 @@ public void setDefaultAllowAll(boolean dfltAllowAll) {
 
     /** {@inheritDoc} */
     @Override public Map<String, Collection<SecurityPermission>> cachePermissions() {
-        return cachePerms;
+        return cachePermissions;
     }
 
     /** {@inheritDoc} */
     @Override public Map<String, Collection<SecurityPermission>> taskPermissions() {
-        return taskPerms;
+        return taskPermissions;
     }
 
     /** {@inheritDoc} */
     @Override public Map<String, Collection<SecurityPermission>> servicePermissions() {
-        return srvcPerms;
+        return servicePermissions;
     }
 
     /** {@inheritDoc} */
     @Nullable @Override public Collection<SecurityPermission> systemPermissions() {
-        return sysPerms;
+        return systemPermissions;
     }
 
     /** {@inheritDoc} */
@@ -117,6 +129,35 @@ public void setDefaultAllowAll(boolean dfltAllowAll) {
         return dfltAllowAll;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+
+        if (!(o instanceof SecurityBasicPermissionSet))
+            return false;
+
+        SecurityBasicPermissionSet other = (SecurityBasicPermissionSet)o;
+
+        return dfltAllowAll == other.dfltAllowAll &&
+            F.eq(cachePermissions, other.cachePermissions) &&
+            F.eq(taskPermissions, other.taskPermissions) &&
+            F.eq(servicePermissions, other.servicePermissions) &&
+            F.eq(systemPermissions, other.systemPermissions);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = (dfltAllowAll ? 1 : 0);
+
+        res = 31 * res + (cachePermissions != null ? cachePermissions.hashCode() : 0);
+        res = 31 * res + (taskPermissions != null ? taskPermissions.hashCode() : 0);
+        res = 31 * res + (servicePermissions != null ? servicePermissions.hashCode() : 0);
+        res = 31 * res + (systemPermissions != null ? systemPermissions.hashCode() : 0);
+
+        return res;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(SecurityBasicPermissionSet.class, this);

From f66c23cbb9a6f2c923ebf75c58f00afaf1c0b5f3 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Wed, 3 May 2017 17:47:45 +0300
Subject: [PATCH 147/516] IGNITE-4939 Receive event before cache initialized
 fix

---
 .../processors/cache/GridCacheProcessor.java  | 20 +++++++++++
 .../cache/IgniteCacheProxyStub.java           | 33 +++++++++++++++++++
 .../CacheContinuousQueryHandler.java          |  3 +-
 3 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyStub.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 0be2072e3766d..5a54e53dbd3f5 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3345,6 +3345,26 @@ public <K, V> IgniteCacheProxy<K, V> jcache(@Nullable String name) {
         return cache;
     }
 
+    /**
+     * @param name Cache name.
+     * @param id Cache id.
+     * @return Cache instance for given name or stub if cache not created yet, but cache descriptor exists.
+     */
+    @SuppressWarnings("unchecked")
+    public <K, V> IgniteCacheProxy<K, V> safeJcache(String name, int id) {
+        assert name != null;
+
+        IgniteCacheProxy<K, V> cache = (IgniteCacheProxy<K, V>)jCacheProxies.get(name);
+
+        if (cache == null)
+            if (cacheDescriptor(id) != null && CU.isSystemCache(name))
+                cache = new IgniteCacheProxyStub<K, V>();
+            else
+                throw new IllegalArgumentException("Cache is not configured: " + name);
+
+        return cache;
+    }
+
     /**
      * @return All configured public cache instances.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyStub.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyStub.java
new file mode 100644
index 0000000000000..75489edb98e74
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyStub.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+/**
+ * Cache proxy stub.
+ */
+public class IgniteCacheProxyStub<K, V> extends IgniteCacheProxy<K, V>{
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * Empty constructor.
+     */
+    public IgniteCacheProxyStub() {
+        // No-op.
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 83edab444f8c0..1a6577d241774 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -716,7 +716,8 @@ private void notifyCallback0(UUID nodeId,
 
         GridCacheContext<K, V> cctx = cacheContext(ctx);
 
-        final IgniteCache cache = cctx.kernalContext().cache().jcache(cctx.name());
+        //returns stub if system cache not created yet
+        final IgniteCache cache = cctx.kernalContext().cache().safeJcache(cctx.name(), cctx.cacheId());
 
         if (internal) {
             if (e.isFiltered())

From 45b4d6316145d0b4b46713409f5e8fbe55ff4c41 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 4 May 2017 12:11:37 +0300
Subject: [PATCH 148/516] IGNITE-4939 Receive event before cache initialized
 fix

---
 .../ignite/internal/processors/cache/GridCacheProcessor.java  | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 5a54e53dbd3f5..ccd7ae0bf500d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3352,9 +3352,7 @@ public <K, V> IgniteCacheProxy<K, V> jcache(@Nullable String name) {
      */
     @SuppressWarnings("unchecked")
     public <K, V> IgniteCacheProxy<K, V> safeJcache(String name, int id) {
-        assert name != null;
-
-        IgniteCacheProxy<K, V> cache = (IgniteCacheProxy<K, V>)jCacheProxies.get(name);
+        IgniteCacheProxy<K, V> cache = (IgniteCacheProxy<K, V>)jCacheProxies.get(maskNull(name));
 
         if (cache == null)
             if (cacheDescriptor(id) != null && CU.isSystemCache(name))

From 075bcfca0ea22633be13cd02647e359ad6fdca16 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 4 May 2017 12:21:04 +0300
Subject: [PATCH 149/516] Fix flacky service deployment tests.

---
 .../GridServiceProcessorAbstractSelfTest.java | 17 ++++++
 ...rviceProcessorMultiNodeConfigSelfTest.java | 46 +++++++--------
 ...GridServiceProcessorMultiNodeSelfTest.java | 56 ++++++++++---------
 3 files changed, 68 insertions(+), 51 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
index 0f79855518e3d..6d91f36aab00b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
@@ -30,7 +30,9 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
+import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
 import org.apache.ignite.internal.util.typedef.CA;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.lang.IgniteFuture;
@@ -519,6 +521,21 @@ protected int actualCount(String svcName, Iterable<ServiceDescriptor> descs) {
         return sum;
     }
 
+    /**
+     * @param srvcName Service name
+     * @param expectedDeps Expected number of service deployments
+     *
+     */
+    protected boolean waitForDeployment(final String srvcName, final int expectedDeps) throws IgniteInterruptedCheckedException {
+        final Ignite g = randomGrid();
+
+        return GridTestUtils.waitForCondition(new GridAbsPredicateX() {
+            @Override public boolean applyx() {
+                return actualCount(srvcName, g.services().serviceDescriptors())  == expectedDeps;
+            }
+        }, 1500);
+    }
+
     /**
      * Counter service.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
index 9da62c0abb702..c78bcaae607ac 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeConfigSelfTest.java
@@ -22,7 +22,6 @@
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -56,7 +55,6 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
 
     /** {@inheritDoc} */
     @Override protected ServiceConfiguration[] services() {
-        List<ServiceConfiguration> cfgs = new ArrayList<>();
 
         ServiceConfiguration cfg = new ServiceConfiguration();
 
@@ -65,6 +63,8 @@ public class GridServiceProcessorMultiNodeConfigSelfTest extends GridServiceProc
         cfg.setTotalCount(1);
         cfg.setService(new DummyService());
 
+        List<ServiceConfiguration> cfgs = new ArrayList<>();
+
         cfgs.add(cfg);
 
         cfg = new ServiceConfiguration();
@@ -202,43 +202,31 @@ public void testDeployLimits() throws Exception {
 
         checkCount(name, g.services().serviceDescriptors(), nodeCount());
 
-        int extraNodes = 2;
-
         CountDownLatch latch = new CountDownLatch(1);
 
         DummyService.exeLatch(name, latch);
 
+        int extraNodes = 2;
+
         startExtraNodes(extraNodes);
 
         try {
             latch.await();
 
+            waitForDeployment(name, nodeCount() + 1);
+
             checkCount(name, g.services().serviceDescriptors(), nodeCount() + 1);
         }
         finally {
             stopExtraNodes(extraNodes);
         }
 
-        assertEquals(name, 1, DummyService.cancelled(name));
-
         waitForDeployment(name, nodeCount());
 
-        checkCount(name, g.services().serviceDescriptors(), nodeCount());
-    }
-
-    /**
-     * @param srvcName Service name
-     * @param expectedDeps Expected number of service deployments
-     *
-     */
-    private boolean waitForDeployment(final String srvcName, final int expectedDeps) throws IgniteInterruptedCheckedException {
-        final Ignite g = randomGrid();
+        // Service can be redeployed when nodes is stopping one-by-one.
+        assertEquals(0, DummyService.started(name) - DummyService.cancelled(name));
 
-        return GridTestUtils.waitForCondition(new GridAbsPredicateX() {
-            @Override public boolean applyx() {
-                return actualCount(srvcName, g.services().serviceDescriptors())  == expectedDeps;
-            }
-        }, 1500);
+        checkCount(name, g.services().serviceDescriptors(), nodeCount());
     }
 
     /**
@@ -281,8 +269,12 @@ private void checkDeployOnEachNodeUpdateTopology(String name) throws Exception {
         try {
             latch.await();
 
-            assertEquals(name, newNodes, DummyService.started(name));
-            assertEquals(name, 0, DummyService.cancelled(name));
+            waitForDeployment(name, nodeCount() + newNodes);
+
+            // Since we start extra nodes, there may be extra start and cancel events,
+            // so we check only the difference between start and cancel and
+            // not start and cancel events individually.
+            assertEquals(name, newNodes,  DummyService.started(name) - DummyService.cancelled(name));
 
             checkCount(name, g.services().serviceDescriptors(), nodeCount() + newNodes);
         }
@@ -314,8 +306,12 @@ private void checkDeployOnEachNodeButClientUpdateTopology(String name) throws Ex
         try {
             latch.await();
 
-            assertEquals(name, servers, DummyService.started(name));
-            assertEquals(name, 0, DummyService.cancelled(name));
+            waitForDeployment(name, nodeCount() + servers);
+
+            // Since we start extra nodes, there may be extra start and cancel events,
+            // so we check only the difference between start and cancel and
+            // not start and cancel events individually.
+            assertEquals(name, servers,  DummyService.started(name) - DummyService.cancelled(name));
 
             checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index d133cf299a756..8ecceb9a65d39 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -92,7 +92,7 @@ public void testAffinityDeployUpdateTopology() throws Exception {
         // Store a cache key.
         g.cache(CACHE_NAME).put(affKey, affKey.toString());
 
-        String name = "serviceAffinityUpdateTopology";
+        final String name = "serviceAffinityUpdateTopology";
 
         IgniteServices svcs = g.services().withAsync();
 
@@ -129,9 +129,7 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
         Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
         try {
-            final int prestartedNodes = nodeCount() + 1;
-
-            String name = "serviceOnEachNodeButClientUpdateTopology";
+            final String name = "serviceOnEachNodeButClientUpdateTopology";
 
             Ignite g = randomGrid();
 
@@ -156,8 +154,8 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
             // Ensure service is deployed
             assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
 
-            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
-            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            assertEquals(name, nodeCount(), DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
 
             int servers = 2;
             int clients = 2;
@@ -171,12 +169,12 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
             try {
                 latch.await();
 
-                // Ensure service is deployed
-                assertNotNull(grid(prestartedNodes + servers - 1)
-                    .services().serviceProxy(name, Service.class, false, 2000));
+                waitForDeployment(name, servers);
 
-                TestCase.assertEquals(name, nodeCount() + servers, DummyService.started(name));
-                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+                // Since we start extra nodes, there may be extra start and cancel events,
+                // so we check only the difference between start and cancel and
+                // not start and cancel events individually.
+                assertEquals(name, nodeCount() + servers,  DummyService.started(name) - DummyService.cancelled(name));
 
                 checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
             }
@@ -197,7 +195,7 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
         Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
 
         try {
-            String name = "serviceOnEachNodeUpdateTopology";
+            final String name = "serviceOnEachNodeUpdateTopology";
 
             Ignite g = randomGrid();
 
@@ -231,8 +229,8 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
             // Ensure service is deployed
             assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
 
-            TestCase.assertEquals(name, prestartedNodes, DummyService.started(name));
-            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            assertEquals(name, prestartedNodes, DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
 
             int servers = 2;
             int clients = 2;
@@ -248,11 +246,13 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
             try {
                 latch.await();
 
-                // Ensure service is deployed
-                assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+                waitForDeployment(name, prestartedNodes + extraNodes);
 
-                TestCase.assertEquals(name, prestartedNodes + extraNodes, DummyService.started(name));
-                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+                // Since we start extra nodes, there may be extra start and cancel events,
+                // so we check only the difference between start and cancel and
+                // not start and cancel events individually.
+                assertEquals(name, prestartedNodes + extraNodes,
+                    DummyService.started(name) - DummyService.cancelled(name));
 
                 checkCount(name, g.services().serviceDescriptors(), prestartedNodes + extraNodes);
             }
@@ -269,7 +269,7 @@ public void testDeployOnEachNodeUpdateTopology() throws Exception {
      * @throws Exception If failed.
      */
     public void testDeployLimits() throws Exception {
-            String name = "serviceWithLimitsUpdateTopology";
+        final String name = "serviceWithLimitsUpdateTopology";
 
             Ignite g = randomGrid();
 
@@ -300,24 +300,28 @@ public void testDeployLimits() throws Exception {
 
             latch.await();
 
-            TestCase.assertEquals(name, nodeCount(), DummyService.started(name));
-            TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+        assertEquals(name, nodeCount(), DummyService.started(name));
+        assertEquals(name, 0, DummyService.cancelled(name));
 
             checkCount(name, g.services().serviceDescriptors(), nodeCount());
 
-            int extraNodes = 2;
-
             latch = new CountDownLatch(1);
 
             DummyService.exeLatch(name, latch);
 
-            startExtraNodes(2);
+        int extraNodes = 2;
+
+        startExtraNodes(extraNodes);
 
             try {
                 latch.await();
 
-                TestCase.assertEquals(name, totalInstances, DummyService.started(name));
-                TestCase.assertEquals(name, 0, DummyService.cancelled(name));
+            waitForDeployment(name, totalInstances);
+
+            // Since we start extra nodes, there may be extra start and cancel events,
+            // so we check only the difference between start and cancel and
+            // not start and cancel events individually.
+            assertEquals(name, totalInstances,  DummyService.started(name) - DummyService.cancelled(name));
 
                 checkCount(name, g.services().serviceDescriptors(), totalInstances);
             }

From 25c06b50d46937cb39534cdf4147b862217289a2 Mon Sep 17 00:00:00 2001
From: rfqu <rfq@list.ru>
Date: Tue, 2 May 2017 19:46:44 +0300
Subject: [PATCH 150/516] ignite-4220 Support statements for JDBC and Cassandra
 store

---
 .../store/cassandra/CassandraCacheStore.java  | 16 ++++-
 .../session/LoadCacheCustomQueryWorker.java   | 26 +++++--
 .../tests/IgnitePersistentStoreTest.java      | 23 ++++--
 .../store/jdbc/CacheAbstractJdbcStore.java    | 72 +++++++++++++------
 .../CacheJdbcPojoStoreAbstractSelfTest.java   | 49 +++++++++++++
 5 files changed, 152 insertions(+), 34 deletions(-)

diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
index e8da3a7c8e511..2e1d3ea70318d 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
@@ -20,6 +20,7 @@
 import com.datastax.driver.core.BoundStatement;
 import com.datastax.driver.core.PreparedStatement;
 import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Statement;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -103,10 +104,19 @@ public CassandraCacheStore(DataSource dataSrc, KeyValuePersistenceSettings setti
             CassandraSession ses = getCassandraSession();
 
             for (Object obj : args) {
-                if (obj == null || !(obj instanceof String) || !((String)obj).trim().toLowerCase().startsWith("select"))
-                    continue;
+                LoadCacheCustomQueryWorker<K, V> task = null;
 
-                futs.add(pool.submit(new LoadCacheCustomQueryWorker<>(ses, (String) obj, controller, log, clo)));
+                if (obj instanceof Statement)
+                    task = new LoadCacheCustomQueryWorker<>(ses, (Statement)obj, controller, log, clo);
+                else if (obj instanceof String) {
+                    String qry = ((String)obj).trim();
+
+                    if (qry.toLowerCase().startsWith("select"))
+                        task = new LoadCacheCustomQueryWorker<>(ses, (String) obj, controller, log, clo);
+                }
+
+                if (task != null)
+                    futs.add(pool.submit(task));
             }
 
             for (Future<?> fut : futs)
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/LoadCacheCustomQueryWorker.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/LoadCacheCustomQueryWorker.java
index d3ace7d436e81..d186b98cf72fa 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/LoadCacheCustomQueryWorker.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/session/LoadCacheCustomQueryWorker.java
@@ -36,8 +36,8 @@ public class LoadCacheCustomQueryWorker<K, V> implements Callable<Void> {
     /** Cassandra session to execute CQL query */
     private final CassandraSession ses;
 
-    /** User query. */
-    private final String qry;
+    /** Statement. */
+    private final Statement stmt;
 
     /** Persistence controller */
     private final PersistenceController ctrl;
@@ -49,12 +49,28 @@ public class LoadCacheCustomQueryWorker<K, V> implements Callable<Void> {
     private final IgniteBiInClosure<K, V> clo;
 
     /**
+     * @param ses Session.
+     * @param qry Query.
+     * @param ctrl Control.
+     * @param log Logger.
      * @param clo Closure for loaded values.
      */
     public LoadCacheCustomQueryWorker(CassandraSession ses, String qry, PersistenceController ctrl,
-        IgniteLogger log, IgniteBiInClosure<K, V> clo) {
+                                      IgniteLogger log, IgniteBiInClosure<K, V> clo) {
+        this(ses, new SimpleStatement(qry.trim().endsWith(";") ? qry : qry + ';'), ctrl, log, clo);
+    }
+
+    /**
+     * @param ses Session.
+     * @param stmt Statement.
+     * @param ctrl Control.
+     * @param log Logger.
+     * @param clo Closure for loaded values.
+     */
+    public LoadCacheCustomQueryWorker(CassandraSession ses, Statement stmt, PersistenceController ctrl,
+                                      IgniteLogger log, IgniteBiInClosure<K, V> clo) {
         this.ses = ses;
-        this.qry = qry.trim().endsWith(";") ? qry : qry + ";";
+        this.stmt = stmt;
         this.ctrl = ctrl;
         this.log = log;
         this.clo = clo;
@@ -70,7 +86,7 @@ public LoadCacheCustomQueryWorker(CassandraSession ses, String qry, PersistenceC
 
             /** {@inheritDoc} */
             @Override public Statement getStatement() {
-                return new SimpleStatement(qry);
+                return stmt;
             }
 
             /** {@inheritDoc} */
diff --git a/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java b/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
index 5da6ba25dabea..51d08855388c6 100644
--- a/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
+++ b/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.tests;
 
+import com.datastax.driver.core.SimpleStatement;
 import java.util.Collection;
 import java.util.Map;
 import org.apache.ignite.Ignite;
@@ -35,6 +36,7 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.Assert;
 import org.springframework.core.io.ClassPathResource;
 
 /**
@@ -346,20 +348,29 @@ public void loadCacheTest() {
         LOGGER.info("Running loadCache test");
 
         try (Ignite ignite = Ignition.start("org/apache/ignite/tests/persistence/pojo/ignite-config.xml")) {
-            IgniteCache<PersonId, Person> personCache3 = ignite.getOrCreateCache(new CacheConfiguration<PersonId, Person>("cache3"));
+            CacheConfiguration<PersonId, Person> ccfg = new CacheConfiguration<>("cache3");
+
+            IgniteCache<PersonId, Person> personCache3 = ignite.getOrCreateCache(ccfg);
+
             int size = personCache3.size(CachePeekMode.ALL);
 
             LOGGER.info("Initial cache size " + size);
 
             LOGGER.info("Loading cache data from Cassandra table");
 
-            personCache3.loadCache(null, new String[] {"select * from test1.pojo_test3 limit 3"});
+            String qry = "select * from test1.pojo_test3 limit 3";
+
+            personCache3.loadCache(null, qry);
 
             size = personCache3.size(CachePeekMode.ALL);
-            if (size != 3) {
-                throw new RuntimeException("Cache data was incorrectly loaded from Cassandra. " +
-                    "Expected number of records is 3, but loaded number of records is " + size);
-            }
+            Assert.assertEquals("Cache data was incorrectly loaded from Cassandra table by '" + qry + "'", 3, size);
+
+            personCache3.clear();
+
+            personCache3.loadCache(null, new SimpleStatement(qry));
+
+            size = personCache3.size(CachePeekMode.ALL);
+            Assert.assertEquals("Cache data was incorrectly loaded from Cassandra table by statement", 3, size);
 
             LOGGER.info("Cache data loaded from Cassandra table");
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
index e7ce52651a868..e211fadf2fb74 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
@@ -81,7 +81,6 @@
 import static org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory.DFLT_PARALLEL_LOAD_CACHE_MINIMUM_THRESHOLD;
 import static org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory.DFLT_WRITE_ATTEMPTS;
 import static org.apache.ignite.cache.store.jdbc.JdbcTypesTransformer.NUMERIC_TYPES;
-import static org.apache.ignite.cache.store.jdbc.JdbcTypesTransformer.NUMERIC_TYPES;
 
 /**
  * Implementation of {@link CacheStore} backed by JDBC.
@@ -753,17 +752,34 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
                         }
                     }))
                         throw new CacheLoaderException("Provided key type is not found in store or cache configuration " +
-                            "[cache=" + U.maskName(cacheName) + ", key=" + keyType + "]");
-
-                    String qry = args[i + 1].toString();
+                            "[cache=" + U.maskName(cacheName) + ", key=" + keyType + ']');
 
                     EntryMapping em = entryMapping(cacheName, typeIdForTypeName(kindForName(keyType), keyType));
 
-                    if (log.isInfoEnabled())
-                        log.info("Started load cache using custom query [cache=" + U.maskName(cacheName) +
-                            ", keyType=" + keyType + ", query=" + qry + "]");
+                    Object arg = args[i + 1];
+
+                    LoadCacheCustomQueryWorker<K, V> task;
+
+                    if (arg instanceof PreparedStatement) {
+                        PreparedStatement stmt = (PreparedStatement)arg;
+
+                        if (log.isInfoEnabled())
+                            log.info("Started load cache using custom statement [cache=" + U.maskName(cacheName) +
+                                      ", keyType=" + keyType + ", stmt=" + stmt + ']');
+
+                        task = new LoadCacheCustomQueryWorker<>(em, stmt, clo);
+                    }
+                    else {
+                        String qry = arg.toString();
+
+                        if (log.isInfoEnabled())
+                              log.info("Started load cache using custom query [cache=" + U.maskName(cacheName) +
+                                  ", keyType=" + keyType + ", query=" + qry + ']');
+
+                        task = new LoadCacheCustomQueryWorker<>(em, qry, clo);
+                    }
 
-                    futs.add(pool.submit(new LoadCacheCustomQueryWorker<>(em, qry, clo)));
+                    futs.add(pool.submit(task));
                 }
             }
             else {
@@ -778,7 +794,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
                     processedKeyTypes.add(keyType);
 
                     if (log.isInfoEnabled())
-                        log.info("Started load cache [cache=" + U.maskName(cacheName) + ", keyType=" + keyType + "]");
+                        log.info("Started load cache [cache=" + U.maskName(cacheName) + ", keyType=" + keyType + ']');
 
                     if (parallelLoadCacheMinThreshold > 0) {
                         Connection conn = null;
@@ -795,7 +811,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
                             if (rs.next()) {
                                 if (log.isDebugEnabled())
                                     log.debug("Multithread loading entries from db [cache=" + U.maskName(cacheName) +
-                                        ", keyType=" + keyType + "]");
+                                        ", keyType=" + keyType + ']');
 
                                 int keyCnt = em.keyCols.size();
 
@@ -824,7 +840,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
                         }
                         catch (SQLException e) {
                             log.warning("Failed to load entries from db in multithreaded mode, will try in single thread " +
-                                "[cache=" + U.maskName(cacheName) + ", keyType=" + keyType + " ]", e);
+                                "[cache=" + U.maskName(cacheName) + ", keyType=" + keyType + ']', e);
                         }
                         finally {
                             U.closeQuiet(conn);
@@ -833,7 +849,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
 
                     if (log.isDebugEnabled())
                         log.debug("Single thread loading entries from db [cache=" + U.maskName(cacheName) +
-                            ", keyType=" + keyType + "]");
+                            ", keyType=" + keyType + ']');
 
                     futs.add(pool.submit(loadCacheFull(em, clo)));
                 }
@@ -860,7 +876,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
         EntryMapping em = entryMapping(session().cacheName(), typeIdForObject(key));
 
         if (log.isDebugEnabled())
-            log.debug("Load value from db [table= " + em.fullTableName() + ", key=" + key + "]");
+            log.debug("Load value from db [table= " + em.fullTableName() + ", key=" + key + ']');
 
         Connection conn = null;
 
@@ -1954,12 +1970,26 @@ private class LoadCacheCustomQueryWorker<K1, V1> implements Callable<Void> {
         /** Entry mapping description. */
         private final EntryMapping em;
 
+        /** User statement. */
+        private PreparedStatement stmt;
+
         /** User query. */
-        private final String qry;
+        private String qry;
 
         /** Closure for loaded values. */
         private final IgniteBiInClosure<K1, V1> clo;
 
+        /**
+         * @param em Entry mapping description.
+         * @param stmt User statement.
+         * @param clo Closure for loaded values.
+         */
+        private LoadCacheCustomQueryWorker(EntryMapping em, PreparedStatement stmt, IgniteBiInClosure<K1, V1> clo) {
+            this.em = em;
+            this.stmt = stmt;
+            this.clo = clo;
+        }
+
         /**
          * @param em Entry mapping description.
          * @param qry User query.
@@ -1975,12 +2005,12 @@ private LoadCacheCustomQueryWorker(EntryMapping em, String qry, IgniteBiInClosur
         @Override public Void call() throws Exception {
             Connection conn = null;
 
-            PreparedStatement stmt = null;
-
             try {
-                conn = openConnection(true);
+                if (stmt == null) {
+                    conn = openConnection(true);
 
-                stmt = conn.prepareStatement(qry);
+                    stmt = conn.prepareStatement(qry);
+                }
 
                 stmt.setFetchSize(dialect.getFetchSize());
 
@@ -2006,9 +2036,11 @@ private LoadCacheCustomQueryWorker(EntryMapping em, String qry, IgniteBiInClosur
                 throw new CacheLoaderException("Failed to execute custom query for load cache", e);
             }
             finally {
-                U.closeQuiet(stmt);
+                if (conn != null) {
+                    U.closeQuiet(stmt);
 
-                U.closeQuiet(conn);
+                    U.closeQuiet(conn);
+                }
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
index 1de44f7243fbb..9e59769503f53 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
@@ -328,6 +328,55 @@ protected void checkCacheLoadWithSql() {
         assertEquals(PERSON_CNT, c1.size());
     }
 
+    /**
+     * Checks that data was loaded correctly with prepared statement.
+     */
+    protected void checkCacheLoadWithStatement() throws SQLException {
+        Connection conn = null;
+
+        PreparedStatement stmt = null;
+
+        try {
+            conn = getConnection();
+
+            conn.setAutoCommit(true);
+
+            String qry = "select id, org_id, name, birthday, gender from Person";
+
+            stmt = conn.prepareStatement(qry);
+
+            IgniteCache<Object, Object> c1 = grid().cache(CACHE_NAME);
+
+            c1.loadCache(null, "org.apache.ignite.cache.store.jdbc.model.PersonKey", stmt);
+
+            assertEquals(PERSON_CNT, c1.size());
+        }
+        finally {
+            U.closeQuiet(stmt);
+
+            U.closeQuiet(conn);
+        }
+
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLoadCacheWithStatement() throws Exception {
+        startTestGrid(false, false, false, false, 512);
+
+        checkCacheLoadWithStatement();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLoadCacheWithStatementTx() throws Exception {
+        startTestGrid(false, false, false, true, 512);
+
+        checkCacheLoadWithStatement();
+    }
+
     /**
      * @throws Exception If failed.
      */

From 7d8d672295e6554a82f63a16b73c330ca6050618 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 5 May 2017 14:29:59 +0300
Subject: [PATCH 151/516] IGNITE-4992: Fix Segmented SQL Index self tests. -
 Fixes #1801.

Signed-off-by: Sergi Vladykin <sergi.vladykin@gmail.com>
(cherry picked from commit 0c0cf2c00355ae2aec340473ed19dc5f0e0ddef0)
---
 ...iteSqlSegmentedIndexMultiNodeSelfTest.java |  28 ++++
 .../IgniteSqlSegmentedIndexSelfTest.java      | 148 ++++++++++--------
 .../IgniteCacheQuerySelfTestSuite.java        |   2 +
 3 files changed, 113 insertions(+), 65 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexMultiNodeSelfTest.java

diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexMultiNodeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexMultiNodeSelfTest.java
new file mode 100644
index 0000000000000..549ea5e043048
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexMultiNodeSelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+/**
+ * Tests for correct distributed queries with index consisted of many segments running on multi-node grid.
+ */
+public class IgniteSqlSegmentedIndexMultiNodeSelfTest extends IgniteSqlSegmentedIndexSelfTest {
+    /** {@inheritDoc} */
+    @Override protected int nodesCount() {
+        return 4;
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
index 800138c1f32ac..f236ebf4cf5c6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
@@ -18,11 +18,13 @@
 package org.apache.ignite.internal.processors.query;
 
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
 import java.util.Set;
 import javax.cache.Cache;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheKeyConfiguration;
@@ -46,8 +48,22 @@ public class IgniteSqlSegmentedIndexSelfTest extends GridCommonAbstractTest {
     private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
 
     /** */
-    private static int QRY_PARALLELISM_LVL = 97;
+    private static final String ORG_CACHE_NAME = "org";
+
+    /** */
+    private static final String PERSON_CAHE_NAME = "pers";
+
+    /** */
+    private static final int ORG_CACHE_SIZE = 500;
+
+    /** */
+    private static final int PERSON_CACHE_SIZE = 1000;
+
+    /** */
+    private static final int ORPHAN_ROWS = 10;
 
+    /** */
+    private static int QRY_PARALLELISM_LVL = 97;
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
@@ -70,9 +86,26 @@ public class IgniteSqlSegmentedIndexSelfTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /** @return number of nodes to be prestarted. */
+    protected int nodesCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGridsMultiThreaded(nodesCount(), false);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
-        stopAllGrids(true);
+        super.afterTest();
+
+        grid(0).destroyCaches(Arrays.asList(PERSON_CAHE_NAME, ORG_CACHE_NAME));
     }
 
     /**
@@ -91,14 +124,11 @@ protected <K, V> CacheConfiguration<K, V> cacheConfig(String name, boolean parti
     }
 
     /**
-     * Run tests on single-node grid
      * @throws Exception If failed.
      */
-    public void testSingleNodeIndexSegmentation() throws Exception {
-        startGridsMultiThreaded(1, true);
-
-        ignite(0).createCache(cacheConfig("pers", true, Integer.class, Person.class));
-        ignite(0).createCache(cacheConfig("org", true, Integer.class, Organization.class));
+    public void testSegmentedIndex() throws Exception {
+        ignite(0).createCache(cacheConfig(PERSON_CAHE_NAME, true, Integer.class, Person.class));
+        ignite(0).createCache(cacheConfig(ORG_CACHE_NAME, true, Integer.class, Organization.class));
 
         fillCache();
 
@@ -109,11 +139,10 @@ public void testSingleNodeIndexSegmentation() throws Exception {
 
     /**
      * Run tests on single-node grid
+     *
      * @throws Exception If failed.
      */
-    public void testSingleNodeIndexSegmentationWithSwapEnabled() throws Exception {
-        startGridsMultiThreaded(1, true);
-
+    public void testSegmentedIndexWithEvictionPolicy() throws Exception {
         final IgniteCache<Object, Object> cache = ignite(0).createCache(cacheConfig("org", true, Integer.class, Organization.class)
             .setOffHeapMaxMemory(-1)
             .setSwapEnabled(true)
@@ -131,30 +160,12 @@ public void testSingleNodeIndexSegmentationWithSwapEnabled() throws Exception {
 
     /**
      * Run tests on multi-node grid
+     *
      * @throws Exception If failed.
      */
-    public void testMultiNodeIndexSegmentation() throws Exception {
-        startGridsMultiThreaded(4, true);
-
-        ignite(0).createCache(cacheConfig("pers", true, Integer.class, Person.class));
-        ignite(0).createCache(cacheConfig("org", true, Integer.class, Organization.class));
-
-        fillCache();
-
-        checkDistributedQueryWithSegmentedIndex();
-
-        checkLocalQueryWithSegmentedIndex();
-    }
-
-    /**
-     * Run tests on multi-node grid
-     * @throws Exception If failed.
-     */
-    public void testMultiNodeSegmentedPartitionedWithReplicated() throws Exception {
-        startGridsMultiThreaded(4, true);
-
-        ignite(0).createCache(cacheConfig("pers", true, Integer.class, Person.class));
-        ignite(0).createCache(cacheConfig("org", false, Integer.class, Organization.class));
+    public void testSegmentedPartitionedWithReplicated() throws Exception {
+        ignite(0).createCache(cacheConfig(PERSON_CAHE_NAME, true, Integer.class, Person.class));
+        ignite(0).createCache(cacheConfig(ORG_CACHE_NAME, false, Integer.class, Organization.class));
 
         fillCache();
 
@@ -165,71 +176,78 @@ public void testMultiNodeSegmentedPartitionedWithReplicated() throws Exception {
 
     /**
      * Check distributed joins.
+     *
      * @throws Exception If failed.
      */
     public void checkDistributedQueryWithSegmentedIndex() throws Exception {
-        IgniteCache<Integer, Person> c1 = ignite(0).cache("pers");
+        for (int i = 0; i < nodesCount(); i++) {
+            IgniteCache<Integer, Person> c1 = ignite(i).cache(PERSON_CAHE_NAME);
 
-        int expectedPersons = 0;
+            int expectedPersons = 0;
 
-        for (Cache.Entry<Integer, Person> e : c1) {
-            final Integer orgId = e.getValue().orgId;
+            for (Cache.Entry<Integer, Person> e : c1) {
+                final Integer orgId = e.getValue().orgId;
 
-            if (10 <= orgId && orgId < 500)
-                expectedPersons++;
-        }
+                // We have as orphan ORG rows as orphan PERSON rows.
+                if (ORPHAN_ROWS <= orgId && orgId < 500)
+                    expectedPersons++;
+            }
 
-        String select0 = "select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o where p.orgId = o._key";
+            String select0 = "select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o where p.orgId = o._key";
 
-        List<List<?>> result = c1.query(new SqlFieldsQuery(select0).setDistributedJoins(true)).getAll();
+            List<List<?>> result = c1.query(new SqlFieldsQuery(select0).setDistributedJoins(true)).getAll();
 
-        assertEquals(expectedPersons, result.size());
+            assertEquals(expectedPersons, result.size());
+        }
     }
 
     /**
      * Test local query.
+     *
      * @throws Exception If failed.
      */
     public void checkLocalQueryWithSegmentedIndex() throws Exception {
-        IgniteCache<Integer, Person> c1 = ignite(0).cache("pers");
-        IgniteCache<Integer, Organization> c2 = ignite(0).cache("org");
+        for (int i = 0; i < nodesCount(); i++) {
+            final Ignite node = ignite(i);
 
-        Set<Integer> localOrgIds = new HashSet<>();
+            IgniteCache<Integer, Person> c1 = node.cache(PERSON_CAHE_NAME);
+            IgniteCache<Integer, Organization> c2 = node.cache(ORG_CACHE_NAME);
 
-        for (Cache.Entry<Integer, Organization> e : c2.localEntries())
-            localOrgIds.add(e.getKey());
+            Set<Integer> localOrgIds = new HashSet<>();
 
-        int expectedPersons = 0;
+            for (Cache.Entry<Integer, Organization> e : c2.localEntries())
+                localOrgIds.add(e.getKey());
 
-        for (Cache.Entry<Integer, Person> e : c1.localEntries()) {
-            final Integer orgId = e.getValue().orgId;
+            int expectedPersons = 0;
 
-            if (localOrgIds.contains(orgId))
-                expectedPersons++;
-        }
+            for (Cache.Entry<Integer, Person> e : c1.localEntries()) {
+                final Integer orgId = e.getValue().orgId;
+
+                if (localOrgIds.contains(orgId))
+                    expectedPersons++;
+            }
 
-        String select0 = "select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o where p.orgId = o._key";
+            String select0 = "select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o where p.orgId = o._key";
 
-        List<List<?>> result = c1.query(new SqlFieldsQuery(select0).setLocal(true)).getAll();
+            List<List<?>> result = c1.query(new SqlFieldsQuery(select0).setLocal(true)).getAll();
 
-        assertEquals(expectedPersons, result.size());
+            assertEquals(expectedPersons, result.size());
+        }
     }
 
     /** */
     private void fillCache() {
-        IgniteCache<Object, Object> c1 = ignite(0).cache("pers");
-
-        IgniteCache<Object, Object> c2 = ignite(0).cache("org");
-
-        final int orgCount = 500;
+        IgniteCache<Object, Object> c1 = ignite(0).cache(PERSON_CAHE_NAME);
+        IgniteCache<Object, Object> c2 = ignite(0).cache(ORG_CACHE_NAME);
 
-        for (int i = 0; i < orgCount; i++)
+        for (int i = 0; i < ORG_CACHE_SIZE; i++)
             c2.put(i, new Organization("org-" + i));
 
         final Random random = new Random();
 
-        for (int i = 0; i < 1000; i++) {
-            int orgID = 10 + random.nextInt(orgCount + 10);
+        for (int i = 0; i < PERSON_CACHE_SIZE; i++) {
+            // We have as orphan ORG rows as orphan PERSON rows.
+            int orgID = ORPHAN_ROWS + random.nextInt(ORG_CACHE_SIZE + ORPHAN_ROWS);
 
             c1.put(i, new Person(orgID, "pers-" + i));
         }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 0c74f127fbff0..4fb729d56c1ed 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -98,6 +98,7 @@
 import org.apache.ignite.internal.processors.cache.query.IndexingSpiQueryTxSelfTest;
 import org.apache.ignite.internal.processors.query.IgniteSqlEntryCacheModeAgnosticTest;
 import org.apache.ignite.internal.processors.query.IgniteSqlSchemaIndexingTest;
+import org.apache.ignite.internal.processors.query.IgniteSqlSegmentedIndexMultiNodeSelfTest;
 import org.apache.ignite.internal.processors.query.IgniteSqlSegmentedIndexSelfTest;
 import org.apache.ignite.internal.processors.query.IgniteSqlSplitterSelfTest;
 import org.apache.ignite.internal.processors.query.h2.GridH2IndexRebuildTest;
@@ -137,6 +138,7 @@ public static TestSuite suite() throws Exception {
         // Queries tests.
         suite.addTestSuite(IgniteSqlSplitterSelfTest.class);
         suite.addTestSuite(IgniteSqlSegmentedIndexSelfTest.class);
+        suite.addTestSuite(IgniteSqlSegmentedIndexMultiNodeSelfTest.class);
         suite.addTestSuite(IgniteSqlSchemaIndexingTest.class);
         suite.addTestSuite(GridCacheQueryIndexDisabledSelfTest.class);
         suite.addTestSuite(IgniteCacheQueryLoadSelfTest.class);

From 61c118f7b8097c554d7e2e4a09133fe3978d73d7 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Wed, 19 Apr 2017 13:55:02 +0300
Subject: [PATCH 152/516] IGNITE-4993 - Fixing distributed joins on segmented
 index.

(cherry picked from commit 800b8bd)
---
 .../query/h2/opt/GridH2IndexBase.java         | 31 ++++++++++---------
 .../IgniteSqlSegmentedIndexSelfTest.java      |  2 +-
 2 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index fc5eb4b35c911..efa95a8979b34 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -406,9 +406,7 @@ protected static IndexingQueryFilter threadLocalFilter() {
 
         GridCacheContext<?, ?> cctx = getTable().rowDescriptor().context();
 
-        boolean isLocal = qctx.distributedJoinMode() == LOCAL_ONLY;
-
-        return new DistributedLookupBatch(cctx, ucast, affColId, isLocal);
+        return new DistributedLookupBatch(cctx, ucast, affColId);
     }
 
     /**
@@ -1078,9 +1076,6 @@ private class DistributedLookupBatch implements IndexLookupBatch {
         /** */
         final int affColId;
 
-        /** */
-        private final boolean localQuery;
-
         /** */
         GridH2QueryContext qctx;
 
@@ -1106,13 +1101,11 @@ private class DistributedLookupBatch implements IndexLookupBatch {
          * @param cctx Cache Cache context.
          * @param ucast Unicast or broadcast query.
          * @param affColId Affinity column ID.
-         * @param localQuery Local query flag.
          */
-        DistributedLookupBatch(GridCacheContext<?, ?> cctx, boolean ucast, int affColId, boolean localQuery) {
+        DistributedLookupBatch(GridCacheContext<?, ?> cctx, boolean ucast, int affColId) {
             this.cctx = cctx;
             this.ucast = ucast;
             this.affColId = affColId;
-            this.localQuery = localQuery;
         }
 
         /**
@@ -1184,25 +1177,26 @@ private Object getAffinityKey(SearchRow firstRow, SearchRow lastRow) {
 
             Object affKey = affColId == -1 ? null : getAffinityKey(firstRow, lastRow);
 
+            boolean locQry = localQuery();
+
             List<SegmentKey> segmentKeys;
-            Future<Cursor> fut;
 
             if (affKey != null) {
                 // Affinity key is provided.
                 if (affKey == EXPLICIT_NULL) // Affinity key is explicit null, we will not find anything.
                     return false;
 
-                segmentKeys = F.asList(rangeSegment(cctx, qctx, affKey, localQuery));
+                segmentKeys = F.asList(rangeSegment(cctx, qctx, affKey, locQry));
             }
             else {
                 // Affinity key is not provided or is not the same in upper and lower bounds, we have to broadcast.
                 if (broadcastSegments == null)
-                    broadcastSegments = broadcastSegments(qctx, cctx, localQuery);
+                    broadcastSegments = broadcastSegments(qctx, cctx, locQry);
 
                 segmentKeys = broadcastSegments;
             }
 
-            if (localQuery && segmentKeys.isEmpty())
+            if (locQry && segmentKeys.isEmpty())
                 return false; // Nothing to do
 
             assert !F.isEmpty(segmentKeys) : segmentKeys;
@@ -1243,7 +1237,7 @@ private Object getAffinityKey(SearchRow firstRow, SearchRow lastRow) {
                     batchFull = true;
             }
 
-            fut = new DoneFuture<>(segmentKeys.size() == 1 ?
+            Future<Cursor> fut = new DoneFuture<>(segmentKeys.size() == 1 ?
                 new UnicastCursor(rangeId, segmentKeys, rangeStreams) :
                 new BroadcastCursor(rangeId, segmentKeys, rangeStreams));
 
@@ -1257,6 +1251,15 @@ private Object getAffinityKey(SearchRow firstRow, SearchRow lastRow) {
             return batchFull;
         }
 
+        /**
+         * @return {@code True} if local query execution is enforced.
+         */
+        private boolean localQuery() {
+            assert qctx != null : "Missing query context: " + this;
+
+            return qctx.distributedJoinMode() == LOCAL_ONLY;
+        }
+
         /**
          *
          */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
index f236ebf4cf5c6..2c21d1a9b8e41 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
@@ -93,7 +93,7 @@ protected int nodesCount() {
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
-        startGridsMultiThreaded(nodesCount(), false);
+        startGrids(nodesCount());
     }
 
     /** {@inheritDoc} */

From 1a4c1b66f7122b4fadc57f6047caf80dfe75e2f4 Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Sat, 6 May 2017 01:21:46 +0300
Subject: [PATCH 153/516] Fix segmented index snapshot usage.

---
 .../query/h2/opt/GridH2IndexBase.java         |  6 +++-
 .../query/h2/opt/GridH2TreeIndex.java         | 20 +++++++------
 .../IgniteSqlSegmentedIndexSelfTest.java      | 28 +++++++++++++++++++
 3 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index efa95a8979b34..8eb082a112b9a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -480,7 +480,11 @@ private void onIndexRangeRequest(final ClusterNode node, final GridH2IndexRangeR
 
                 if (msg.bounds() != null) {
                     // This is the first request containing all the search rows.
-                    ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> snapshot0 = qctx.getSnapshot(idxId);
+                    Object[] snapshotObj = qctx.getSnapshot(idxId);
+
+                    // We should get correct segment here.
+                    ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> snapshot0 = snapshotObj == null ? null :
+                        (ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row>)snapshotObj[msg.segment()];
 
                     assert !msg.bounds().isEmpty() : "empty bounds";
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
index 663d86386b8de..2baeca9be4d7f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
@@ -166,13 +166,17 @@ public GridH2TreeIndex(String name, GridH2Table tbl, boolean pk, List<IndexColum
     @Override protected Object doTakeSnapshot() {
         assert snapshotEnabled;
 
-        int seg = threadLocalSegment();
+        final ConcurrentNavigableMap[] snapshot = new ConcurrentNavigableMap[segmentsCount()];
+
+        for(int i = 0; i < segmentsCount(); i++) {
+            final ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> tree = segments[i];
 
-        ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> tree = segments[seg];
+            snapshot[i] = tree instanceof SnapTreeMap ?
+                ((SnapTreeMap)tree).clone() :
+                ((GridOffHeapSnapTreeMap)tree).clone();
+        }
 
-        return tree instanceof SnapTreeMap ?
-            ((SnapTreeMap)tree).clone() :
-            ((GridOffHeapSnapTreeMap)tree).clone();
+        return snapshot;
     }
 
     /** {@inheritDoc} */
@@ -180,12 +184,12 @@ protected ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> treeForRead(in
         if (!snapshotEnabled)
             return segments[seg];
 
-        ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> res = threadLocalSnapshot();
+        final ConcurrentNavigableMap[] snapshot = threadLocalSnapshot();
 
-        if (res == null)
+        if (snapshot == null)
             return segments[seg];
 
-        return res;
+        return snapshot[seg];
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
index 2c21d1a9b8e41..7c4fe9c790e57 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheKeyConfiguration;
+import org.apache.ignite.cache.CacheMemoryMode;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
@@ -137,6 +138,33 @@ public void testSegmentedIndex() throws Exception {
         checkLocalQueryWithSegmentedIndex();
     }
 
+    /**
+     * Check correct index snapshots with segmented indices.
+     * @throws Exception If failed.
+     */
+    public void testSegmentedIndexReproducableResults() throws Exception {
+        ignite(0).createCache(cacheConfig(ORG_CACHE_NAME, true, Integer.class, Organization.class)
+        .setOffHeapMaxMemory(-1) // Make index snapshot to be used.
+        .setMemoryMode(CacheMemoryMode.OFFHEAP_TIERED));
+
+        IgniteCache<Object, Object> cache = ignite(0).cache(ORG_CACHE_NAME);
+
+        // Unequal entries distribution among partitions.
+        int expectedSize = nodesCount() * QRY_PARALLELISM_LVL *  3 / 2;
+
+        for (int i = 0; i < expectedSize; i++)
+            cache.put(i, new Organization("org-" + i));
+
+        String select0 = "select * from \"org\".Organization o";
+
+        // Check for stable results.
+        for(int i = 0; i < 10; i++) {
+            List<List<?>> result = cache.query(new SqlFieldsQuery(select0)).getAll();
+
+            assertEquals(expectedSize, result.size());
+        }
+    }
+
     /**
      * Run tests on single-node grid
      *

From 2f832a3621432ba3947c8d519ff124b29eb8d9a0 Mon Sep 17 00:00:00 2001
From: Sergi Vladykin <sergi.vladykin@gmail.com>
Date: Tue, 9 May 2017 06:11:34 +0300
Subject: [PATCH 154/516] GG-12182 Incorrect splitting of aggregate subqueries
 in SELECT

cheerry picked from commit d10091d6f2
---
 .../query/h2/sql/GridSqlQuerySplitter.java    |  7 ++++-
 .../query/IgniteSqlSplitterSelfTest.java      | 30 +++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index aec0b3664f47b..5bd7f2b9772b6 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -1751,13 +1751,18 @@ private static <Z> void set(List<Z> list, int idx, Z item) {
     }
 
     /**
-     * @param el Expression.
+     * @param el Expression part in SELECT clause.
      * @return {@code true} If expression contains aggregates.
      */
     private static boolean hasAggregates(GridSqlAst el) {
         if (el instanceof GridSqlAggregateFunction)
             return true;
 
+        // If in SELECT clause we have a subquery expression with aggregate,
+        // we should not split it. Run the whole subquery on MAP stage.
+        if (el instanceof GridSqlQuery)
+            return false;
+
         for (int i = 0; i < el.size(); i++) {
             if (hasAggregates(el.child(i)))
                 return true;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 1aa7e7215b972..8daf97d50b69a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.affinity.AffinityKey;
 import org.apache.ignite.cache.affinity.AffinityKeyMapped;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
@@ -182,6 +183,35 @@ public void testReplicatedOnlyTables() {
         }
     }
 
+    /**
+     */
+    public void testSubQueryWithAggregate() {
+        CacheConfiguration ccfg1 = cacheConfig("pers", true,
+            AffinityKey.class, Person2.class);
+
+        IgniteCache<AffinityKey<Integer>, Person2> c1 = ignite(0).getOrCreateCache(ccfg1);
+
+        try {
+            int orgId = 100500;
+
+            c1.put(new AffinityKey<>(1, orgId), new Person2(orgId, "Vasya"));
+            c1.put(new AffinityKey<>(2, orgId), new Person2(orgId, "Another Vasya"));
+
+            List<List<?>> rs = c1.query(new SqlFieldsQuery("select name, " +
+                "(select count(1) from Person2 q where q.orgId = p.orgId) " +
+                "from Person2 p order by name desc")).getAll();
+
+            assertEquals(2, rs.size());
+            assertEquals("Vasya", rs.get(0).get(0));
+            assertEquals(2L, rs.get(0).get(1));
+            assertEquals("Another Vasya", rs.get(1).get(0));
+            assertEquals(2L, rs.get(1).get(1));
+        }
+        finally {
+            c1.destroy();
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From d3b2e99a1e2c580f156d89185414abcd0ebcd729 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 12 May 2017 19:28:36 +0300
Subject: [PATCH 155/516] Fixed segmented indices snapshots.

(cherry picked from commit b807e4b)
---
 .../query/h2/opt/GridH2IndexBase.java         |  6 +-
 .../processors/query/h2/opt/GridH2Table.java  | 80 ++++++++++++-------
 .../query/h2/opt/GridH2TreeIndex.java         | 16 ++--
 .../h2/twostep/GridMapQueryExecutor.java      |  2 +-
 .../IgniteSqlSegmentedIndexSelfTest.java      |  2 +-
 5 files changed, 59 insertions(+), 47 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 8eb082a112b9a..efa95a8979b34 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -480,11 +480,7 @@ private void onIndexRangeRequest(final ClusterNode node, final GridH2IndexRangeR
 
                 if (msg.bounds() != null) {
                     // This is the first request containing all the search rows.
-                    Object[] snapshotObj = qctx.getSnapshot(idxId);
-
-                    // We should get correct segment here.
-                    ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> snapshot0 = snapshotObj == null ? null :
-                        (ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row>)snapshotObj[msg.segment()];
+                    ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> snapshot0 = qctx.getSnapshot(idxId);
 
                     assert !msg.bounds().isEmpty() : "empty bounds";
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
index 4d5ea4bfec9e2..6b087c2cdf91e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
@@ -25,6 +25,7 @@
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -74,10 +75,10 @@ public class GridH2Table extends TableBase {
     private boolean destroyed;
 
     /** */
-    private final Set<Session> sessions = Collections.newSetFromMap(new ConcurrentHashMap8<Session,Boolean>());
+    private final Set<Session> sessions = Collections.newSetFromMap(new ConcurrentHashMap8<Session, Boolean>());
 
     /** */
-    private final AtomicReference<Object[]> actualSnapshot = new AtomicReference<>();
+    private final AtomicReferenceArray<Object[]> actualSnapshot;
 
     /** */
     private IndexColumn affKeyCol;
@@ -141,6 +142,11 @@ public GridH2Table(CreateTableData createTblData, @Nullable GridH2RowDescriptor
 
         snapshotEnabled = desc == null || desc.snapshotableIndex();
 
+        final int segments = desc != null ? desc.configuration().getQueryParallelism() :
+            index(1).segmentsCount(); // Get index segments count from PK index. Null desc can be passed from tests.
+
+        actualSnapshot = snapshotEnabled ? new AtomicReferenceArray<Object[]>(Math.max(segments, 1)) : null;
+
         lock = snapshotEnabled ? new ReentrantReadWriteLock() : null;
     }
 
@@ -259,11 +265,22 @@ private boolean onSwapUnswap(CacheObject key, @Nullable CacheObject val) throws
         }
 
         if (snapshotInLock())
-            snapshotIndexes(null);
+            snapshotIndexes(null, threadLocalSegmentId());
 
         return false;
     }
 
+    /**
+     * @return segmentId for current thread.
+     */
+    private int threadLocalSegmentId() {
+        final GridH2QueryContext qctx = GridH2QueryContext.get();
+
+        assert qctx != null;
+
+        return qctx.segment();
+    }
+
     /**
      * @return {@code True} If we must snapshot and release index snapshots in {@link #lock(Session, boolean, boolean)}
      * and {@link #unlock(Session)} methods.
@@ -281,22 +298,22 @@ private boolean snapshotInLock() {
     /**
      * @param qctx Query context.
      */
-    public void snapshotIndexes(GridH2QueryContext qctx) {
+    public void snapshotIndexes(GridH2QueryContext qctx, int segment) {
         if (!snapshotEnabled)
             return;
 
-        Object[] snapshots;
+        Object[] segmentSnapshot;
 
         Lock l;
 
         // Try to reuse existing snapshots outside of the lock.
-        for (long waitTime = 200;; waitTime *= 2) { // Increase wait time to avoid starvation.
-            snapshots = actualSnapshot.get();
+        for (long waitTime = 200; ; waitTime *= 2) { // Increase wait time to avoid starvation.
+            segmentSnapshot = actualSnapshot.get(segment);
 
-            if (snapshots != null) { // Reuse existing snapshot without locking.
-                snapshots = doSnapshotIndexes(snapshots, qctx);
+            if (segmentSnapshot != null) { // Reuse existing snapshot without locking.
+                segmentSnapshot = doSnapshotIndexes(segmentSnapshot, qctx);
 
-                if (snapshots != null)
+                if (segmentSnapshot != null)
                     return; // Reused successfully.
             }
 
@@ -308,17 +325,17 @@ public void snapshotIndexes(GridH2QueryContext qctx) {
 
         try {
             // Try again inside of the lock.
-            snapshots = actualSnapshot.get();
+            segmentSnapshot = actualSnapshot.get(segment);
 
-            if (snapshots != null) // Try reusing.
-                snapshots = doSnapshotIndexes(snapshots, qctx);
+            if (segmentSnapshot != null) // Try reusing.
+                segmentSnapshot = doSnapshotIndexes(segmentSnapshot, qctx);
 
-            if (snapshots == null) { // Reuse failed, produce new snapshots.
-                snapshots = doSnapshotIndexes(null, qctx);
+            if (segmentSnapshot == null) { // Reuse failed, produce new snapshots.
+                segmentSnapshot = doSnapshotIndexes(null, qctx);
 
-                assert snapshots != null;
+                assert segmentSnapshot != null;
 
-                actualSnapshot.set(snapshots);
+                actualSnapshot.set(segment, segmentSnapshot);
             }
         }
         finally {
@@ -373,19 +390,20 @@ public Lock lock(boolean exclusive, long waitMillis) {
      * Must be called inside of write lock because when using multiple indexes we have to ensure that all of them have
      * the same contents at snapshot taking time.
      *
+     * @param segmentSnapshot snapshot to be reused.
      * @param qctx Query context.
      * @return New indexes data snapshot.
      */
     @SuppressWarnings("unchecked")
-    private Object[] doSnapshotIndexes(Object[] snapshots, GridH2QueryContext qctx) {
+    private Object[] doSnapshotIndexes(Object[] segmentSnapshot, GridH2QueryContext qctx) {
         assert snapshotEnabled;
 
-        if (snapshots == null) // Nothing to reuse, create new snapshots.
-            snapshots = new Object[idxs.size() - 1];
+        if (segmentSnapshot == null) // Nothing to reuse, create new snapshots.
+            segmentSnapshot = new Object[idxs.size() - 1];
 
         // Take snapshots on all except first which is scan.
         for (int i = 1, len = idxs.size(); i < len; i++) {
-            Object s = snapshots[i - 1];
+            Object s = segmentSnapshot[i - 1];
 
             boolean reuseExisting = s != null;
 
@@ -400,15 +418,15 @@ private Object[] doSnapshotIndexes(Object[] snapshots, GridH2QueryContext qctx)
                     index(j).releaseSnapshot();
 
                 // Drop invalidated snapshot.
-                actualSnapshot.compareAndSet(snapshots, null);
+                actualSnapshot.compareAndSet(threadLocalSegmentId(), segmentSnapshot, null);
 
                 return null;
             }
 
-            snapshots[i - 1] = s;
+            segmentSnapshot[i - 1] = s;
         }
 
-        return snapshots;
+        return segmentSnapshot;
     }
 
     /** {@inheritDoc} */
@@ -585,7 +603,7 @@ else if (old != null) // Row was not replaced, need to remove manually.
                     for (int i = 2, len = idxs.size(); i < len; i++) {
                         Row res = index(i).remove(old);
 
-                        assert eq(pk, res, old): "\n" + old + "\n" + res + "\n" + i + " -> " + index(i).getName();
+                        assert eq(pk, res, old) : "\n" + old + "\n" + res + "\n" + i + " -> " + index(i).getName();
                     }
                 }
                 else
@@ -593,7 +611,7 @@ else if (old != null) // Row was not replaced, need to remove manually.
             }
 
             // The snapshot is not actual after update.
-            actualSnapshot.set(null);
+            actualSnapshot.set(pk.segmentForRow(row), null);
 
             return true;
         }
@@ -625,7 +643,7 @@ private static boolean eq(Index pk, SearchRow r1, SearchRow r2) {
     ArrayList<GridH2IndexBase> indexes() {
         ArrayList<GridH2IndexBase> res = new ArrayList<>(idxs.size() - 1);
 
-        for (int i = 1, len = idxs.size(); i < len ; i++)
+        for (int i = 1, len = idxs.size(); i < len; i++)
             res.add(index(i));
 
         return res;
@@ -643,7 +661,9 @@ public void rebuildIndexes() {
         ArrayList<Index> idxs0 = new ArrayList<>(idxs);
 
         try {
-            snapshotIndexes(null); // Allow read access while we are rebuilding indexes.
+            // Allow read access while we are rebuilding indexes.
+            for (int seg = 0; seg < actualSnapshot.length(); seg++)
+                snapshotIndexes(null, seg);
 
             for (int i = 1, len = idxs.size(); i < len; i++) {
                 GridH2IndexBase newIdx = index(i).rebuild();
@@ -804,8 +824,8 @@ public static class Engine implements TableEngine {
          * @param desc Row descriptor.
          * @param factory Indexes factory.
          * @param space Space name.
-         * @throws SQLException If failed.
          * @return Created table.
+         * @throws SQLException If failed.
          */
         public static synchronized GridH2Table createTable(Connection conn, String sql,
             @Nullable GridH2RowDescriptor desc, IndexesFactory factory, String space)
@@ -867,7 +887,7 @@ public ScanIndex(GridH2IndexBase delegate) {
             double baseCost = getCostRangeIndex(masks, rows, filters, filter, sortOrder, true);
             int mul = delegate.getDistributedMultiplier(ses, filters, filter);
 
-            return  mul * baseCost;
+            return mul * baseCost;
         }
 
         /** {@inheritDoc} */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
index 2baeca9be4d7f..d4970bd2e1e9e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
@@ -166,30 +166,26 @@ public GridH2TreeIndex(String name, GridH2Table tbl, boolean pk, List<IndexColum
     @Override protected Object doTakeSnapshot() {
         assert snapshotEnabled;
 
-        final ConcurrentNavigableMap[] snapshot = new ConcurrentNavigableMap[segmentsCount()];
+        int seg = threadLocalSegment();
 
-        for(int i = 0; i < segmentsCount(); i++) {
-            final ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> tree = segments[i];
+        ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> tree = segments[seg];
 
-            snapshot[i] = tree instanceof SnapTreeMap ?
+        return tree instanceof SnapTreeMap ?
                 ((SnapTreeMap)tree).clone() :
                 ((GridOffHeapSnapTreeMap)tree).clone();
         }
 
-        return snapshot;
-    }
-
     /** {@inheritDoc} */
     protected ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> treeForRead(int seg) {
         if (!snapshotEnabled)
             return segments[seg];
 
-        final ConcurrentNavigableMap[] snapshot = threadLocalSnapshot();
+        ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> res = threadLocalSnapshot();
 
-        if (snapshot == null)
+        if (res == null)
             return segments[seg];
 
-        return snapshot[seg];
+        return res;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 6416b21d8e18e..fdf27cd2fa3aa 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -582,7 +582,7 @@ private void onQueryRequest0(
 
                     Objects.requireNonNull(tbl, identifier);
 
-                    tbl.snapshotIndexes(qctx);
+                    tbl.snapshotIndexes(qctx, segmentId);
 
                     snapshotedTbls.add(tbl);
                 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
index 7c4fe9c790e57..951b42d858d11 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
@@ -144,7 +144,7 @@ public void testSegmentedIndex() throws Exception {
      */
     public void testSegmentedIndexReproducableResults() throws Exception {
         ignite(0).createCache(cacheConfig(ORG_CACHE_NAME, true, Integer.class, Organization.class)
-        .setOffHeapMaxMemory(-1) // Make index snapshot to be used.
+        .setOffHeapMaxMemory(-1) // Force index snapshots enabling.
         .setMemoryMode(CacheMemoryMode.OFFHEAP_TIERED));
 
         IgniteCache<Object, Object> cache = ignite(0).cache(ORG_CACHE_NAME);

From 987c182686962673e70398395cb27e94f894713b Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Mon, 15 May 2017 11:54:16 +0300
Subject: [PATCH 156/516] Fixed "IGNITE-5214 ConcurrentModificationException
 with enable DEBUG log level"

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../continuous/CacheContinuousQueryHandler.java | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 1a6577d241774..5f00d580f4231 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -998,6 +998,7 @@ private static class PartitionRecovery {
 
             synchronized (pendingEvts) {
                 if (log.isDebugEnabled()) {
+
                     log.debug("Handling event [lastFiredEvt=" + lastFiredEvt +
                         ", curTop=" + curTop +
                         ", entUpdCnt=" + entry.updateCounter() +
@@ -1126,15 +1127,15 @@ private static class PartitionRecovery {
                             break;
                     }
                 }
-            }
 
-            if (log.isDebugEnabled()) {
-                log.debug("Will send to listener the following events [entries=" + entries +
-                    ", lastFiredEvt=" + lastFiredEvt +
-                    ", curTop=" + curTop +
-                    ", entUpdCnt=" + entry.updateCounter() +
-                    ", partId=" + entry.partition() +
-                    ", pendingEvts=" + pendingEvts + ']');
+                if (log.isDebugEnabled()) {
+                    log.debug("Will send to listener the following events [entries=" + entries +
+                        ", lastFiredEvt=" + lastFiredEvt +
+                        ", curTop=" + curTop +
+                        ", entUpdCnt=" + entry.updateCounter() +
+                        ", partId=" + entry.partition() +
+                        ", pendingEvts=" + pendingEvts + ']');
+                }
             }
 
             return entries;

From 350734f8f9ea8d54626992248b9a1b4a15d758c1 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 15 May 2017 17:39:52 +0300
Subject: [PATCH 157/516] IGNITE-5225: Fixed thread pools incorrect shutdown.

(cherry picked from commit 66cef22)
---
 .../processors/cache/GridCacheAdapter.java    | 12 ++---
 .../ignite/internal/util/IgniteUtils.java     | 45 ++++++++++---------
 2 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index aaea4f08eacb0..67ae9608b0f65 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1132,14 +1132,14 @@ public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, bool
         if (!F.isEmpty(jobs)) {
             ExecutorService execSvc = null;
 
-            if (jobs.size() > 1) {
-                execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
+            try {
+                if (jobs.size() > 1) {
+                    execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
 
-                for (int i = 1; i < jobs.size(); i++)
-                    execSvc.execute(jobs.get(i));
-            }
+                    for (int i = 1; i < jobs.size(); i++)
+                        execSvc.execute(jobs.get(i));
+                }
 
-            try {
                 jobs.get(0).run();
             }
             finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 74e4450c218cb..8ee794188c827 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1873,33 +1873,36 @@ public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
 
         ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, addrs.size()));
 
-        for (final InetAddress addr : addrs) {
-            futs.add(executor.submit(new Runnable() {
-                @Override public void run() {
-                    if (reachable(addr, reachTimeout)) {
-                        synchronized (res) {
-                            res.add(addr);
+        try {
+            for (final InetAddress addr : addrs) {
+                futs.add(executor.submit(new Runnable() {
+                    @Override public void run() {
+                        if (reachable(addr, reachTimeout)) {
+                            synchronized (res) {
+                                res.add(addr);
+                            }
                         }
                     }
-                }
-            }));
-        }
-
-        for (Future<?> fut : futs) {
-            try {
-                fut.get();
+                }));
             }
-            catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
 
-                throw new IgniteException("Thread has been interrupted.", e);
-            }
-            catch (ExecutionException e) {
-                throw new IgniteException(e);
+            for (Future<?> fut : futs) {
+                try {
+                    fut.get();
+                }
+                catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+
+                    throw new IgniteException("Thread has been interrupted.", e);
+                }
+                catch (ExecutionException e) {
+                    throw new IgniteException(e);
+                }
             }
         }
-
-        executor.shutdown();
+        finally {
+            executor.shutdown();
+        }
 
         return res;
     }

From 00cb5dddb3c56a41efdf2daf80a3de0bfed68678 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 15 May 2017 20:24:10 +0300
Subject: [PATCH 158/516] IGNITE-5225: Fix NPE caused by changes in
 IGNITE-4577.

(cherry picked from commit d463840)
---
 .../ignite/internal/util/IgniteUtils.java       |  4 ++--
 .../communication/tcp/TcpCommunicationSpi.java  | 17 +++++++++++------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 8ee794188c827..7c250b6c10c93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1853,11 +1853,11 @@ public static synchronized boolean isLocalHostChanged() throws IOException {
      * @return List of reachable addresses.
      */
     public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
-        final int reachTimeout = 2000;
-
         if (addrs.isEmpty())
             return Collections.emptyList();
 
+        final int reachTimeout = 2000;
+
         if (addrs.size() == 1) {
             InetAddress addr = F.first(addrs);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 765b6151e6ce7..9b45982ff893a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2769,22 +2769,27 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
         Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
 
-        for (InetSocketAddress addr : addrs)
-            allInetAddrs.add(addr.getAddress());
+        for (InetSocketAddress addr : addrs) {
+            // Skip unresolved as addr.getAddress() can return null.
+            if(!addr.isUnresolved())
+                allInetAddrs.add(addr.getAddress());
+        }
 
         List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
 
         if (reachableInetAddrs.size() < allInetAddrs.size()) {
             LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
 
+            List<InetSocketAddress> unreachableInetAddr = new ArrayList<>(allInetAddrs.size() - reachableInetAddrs.size());
+
             for (InetSocketAddress addr : addrs) {
                 if (reachableInetAddrs.contains(addr.getAddress()))
                     addrs0.add(addr);
+                else
+                    unreachableInetAddr.add(addr);
             }
-            for (InetSocketAddress addr : addrs) {
-                if (!reachableInetAddrs.contains(addr.getAddress()))
-                    addrs0.add(addr);
-            }
+
+            addrs0.addAll(unreachableInetAddr);
 
             addrs = addrs0;
         }

From a5d2b9ba3ed1e72ef3970ecfed499a97f925ff2c Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 15 May 2017 17:39:52 +0300
Subject: [PATCH 159/516] Fixed thread pools incorrect shutdown.

(cherry picked from commit dacf973)
---
 .../processors/cache/GridCacheAdapter.java    | 12 ++---
 .../ignite/internal/util/IgniteUtils.java     | 45 ++++++++++---------
 2 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index a50661e137600..7a35a2abaa303 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1132,14 +1132,14 @@ public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, bool
         if (!F.isEmpty(jobs)) {
             ExecutorService execSvc = null;
 
-            if (jobs.size() > 1) {
-                execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
+            try {
+                if (jobs.size() > 1) {
+                    execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
 
-                for (int i = 1; i < jobs.size(); i++)
-                    execSvc.execute(jobs.get(i));
-            }
+                    for (int i = 1; i < jobs.size(); i++)
+                        execSvc.execute(jobs.get(i));
+                }
 
-            try {
                 jobs.get(0).run();
             }
             finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index a204e9ac729a5..cc21b86db7c94 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1867,33 +1867,36 @@ public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
 
         ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, addrs.size()));
 
-        for (final InetAddress addr : addrs) {
-            futs.add(executor.submit(new Runnable() {
-                @Override public void run() {
-                    if (reachable(addr, reachTimeout)) {
-                        synchronized (res) {
-                            res.add(addr);
+        try {
+            for (final InetAddress addr : addrs) {
+                futs.add(executor.submit(new Runnable() {
+                    @Override public void run() {
+                        if (reachable(addr, reachTimeout)) {
+                            synchronized (res) {
+                                res.add(addr);
+                            }
                         }
                     }
-                }
-            }));
-        }
-
-        for (Future<?> fut : futs) {
-            try {
-                fut.get();
+                }));
             }
-            catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
 
-                throw new IgniteException("Thread has been interrupted.", e);
-            }
-            catch (ExecutionException e) {
-                throw new IgniteException(e);
+            for (Future<?> fut : futs) {
+                try {
+                    fut.get();
+                }
+                catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+
+                    throw new IgniteException("Thread has been interrupted.", e);
+                }
+                catch (ExecutionException e) {
+                    throw new IgniteException(e);
+                }
             }
         }
-
-        executor.shutdown();
+        finally {
+            executor.shutdown();
+        }
 
         return res;
     }

From 5e08c2951522a640bf84b6c60f03f29edd970ffc Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 15 May 2017 20:24:10 +0300
Subject: [PATCH 160/516] IGNITE-5225: Fix NPE caused by changes in
 IGNITE-4577.

(cherry picked from commit d463840)
---
 .../ignite/internal/util/IgniteUtils.java       |  4 ++--
 .../communication/tcp/TcpCommunicationSpi.java  | 17 +++++++++++------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index cc21b86db7c94..f8f1fa5bddfa0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1847,11 +1847,11 @@ public static synchronized boolean isLocalHostChanged() throws IOException {
      * @return List of reachable addresses.
      */
     public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
-        final int reachTimeout = 2000;
-
         if (addrs.isEmpty())
             return Collections.emptyList();
 
+        final int reachTimeout = 2000;
+
         if (addrs.size() == 1) {
             InetAddress addr = F.first(addrs);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 89ecc36d2ace1..60d97348d42cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2769,22 +2769,27 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
         Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
 
-        for (InetSocketAddress addr : addrs)
-            allInetAddrs.add(addr.getAddress());
+        for (InetSocketAddress addr : addrs) {
+            // Skip unresolved as addr.getAddress() can return null.
+            if(!addr.isUnresolved())
+                allInetAddrs.add(addr.getAddress());
+        }
 
         List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
 
         if (reachableInetAddrs.size() < allInetAddrs.size()) {
             LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
 
+            List<InetSocketAddress> unreachableInetAddr = new ArrayList<>(allInetAddrs.size() - reachableInetAddrs.size());
+
             for (InetSocketAddress addr : addrs) {
                 if (reachableInetAddrs.contains(addr.getAddress()))
                     addrs0.add(addr);
+                else
+                    unreachableInetAddr.add(addr);
             }
-            for (InetSocketAddress addr : addrs) {
-                if (!reachableInetAddrs.contains(addr.getAddress()))
-                    addrs0.add(addr);
-            }
+
+            addrs0.addAll(unreachableInetAddr);
 
             addrs = addrs0;
         }

From 62765b5b7598ab2a82b9c595e4a1276d9f2f5860 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 16 May 2017 11:30:29 +0300
Subject: [PATCH 161/516] DirectByteBufferStreamImpl: converted asserts into
 exceptions.

(cherry picked from commit 560ef60)
---
 .../v2/DirectByteBufferStreamImplV2.java      | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
index d7dc990848094..af45263e36a8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
@@ -1,4 +1,4 @@
-/*
+ /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -27,6 +27,7 @@
 import java.util.Map;
 import java.util.RandomAccess;
 import java.util.UUID;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.direct.stream.DirectByteBufferStream;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -80,7 +81,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<byte[]> BYTE_ARR_CREATOR = new ArrayCreator<byte[]>() {
         @Override public byte[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid byte array length: " + len);
 
             switch (len) {
                 case 0:
@@ -95,7 +97,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<short[]> SHORT_ARR_CREATOR = new ArrayCreator<short[]>() {
         @Override public short[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid short array length: " + len);
 
             switch (len) {
                 case 0:
@@ -110,7 +113,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<int[]> INT_ARR_CREATOR = new ArrayCreator<int[]>() {
         @Override public int[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid int array length: " + len);
 
             switch (len) {
                 case 0:
@@ -125,7 +129,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<long[]> LONG_ARR_CREATOR = new ArrayCreator<long[]>() {
         @Override public long[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid long array length: " + len);
 
             switch (len) {
                 case 0:
@@ -140,7 +145,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<float[]> FLOAT_ARR_CREATOR = new ArrayCreator<float[]>() {
         @Override public float[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid float array length: " + len);
 
             switch (len) {
                 case 0:
@@ -155,7 +161,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<double[]> DOUBLE_ARR_CREATOR = new ArrayCreator<double[]>() {
         @Override public double[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid double array length: " + len);
 
             switch (len) {
                 case 0:
@@ -170,7 +177,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<char[]> CHAR_ARR_CREATOR = new ArrayCreator<char[]>() {
         @Override public char[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid char array length: " + len);
 
             switch (len) {
                 case 0:
@@ -185,7 +193,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<boolean[]> BOOLEAN_ARR_CREATOR = new ArrayCreator<boolean[]>() {
         @Override public boolean[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid boolean array length: " + len);
 
             switch (len) {
                 case 0:

From d7963cb7c5ec5965ec071f2dc603540290e5445c Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 16 May 2017 11:30:29 +0300
Subject: [PATCH 162/516] DirectByteBufferStreamImpl: converted asserts into
 exceptions.

(cherry picked from commit 560ef60)
---
 .../v2/DirectByteBufferStreamImplV2.java      | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
index d7dc990848094..af45263e36a8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
@@ -1,4 +1,4 @@
-/*
+ /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -27,6 +27,7 @@
 import java.util.Map;
 import java.util.RandomAccess;
 import java.util.UUID;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.direct.stream.DirectByteBufferStream;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -80,7 +81,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<byte[]> BYTE_ARR_CREATOR = new ArrayCreator<byte[]>() {
         @Override public byte[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid byte array length: " + len);
 
             switch (len) {
                 case 0:
@@ -95,7 +97,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<short[]> SHORT_ARR_CREATOR = new ArrayCreator<short[]>() {
         @Override public short[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid short array length: " + len);
 
             switch (len) {
                 case 0:
@@ -110,7 +113,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<int[]> INT_ARR_CREATOR = new ArrayCreator<int[]>() {
         @Override public int[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid int array length: " + len);
 
             switch (len) {
                 case 0:
@@ -125,7 +129,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<long[]> LONG_ARR_CREATOR = new ArrayCreator<long[]>() {
         @Override public long[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid long array length: " + len);
 
             switch (len) {
                 case 0:
@@ -140,7 +145,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<float[]> FLOAT_ARR_CREATOR = new ArrayCreator<float[]>() {
         @Override public float[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid float array length: " + len);
 
             switch (len) {
                 case 0:
@@ -155,7 +161,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<double[]> DOUBLE_ARR_CREATOR = new ArrayCreator<double[]>() {
         @Override public double[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid double array length: " + len);
 
             switch (len) {
                 case 0:
@@ -170,7 +177,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<char[]> CHAR_ARR_CREATOR = new ArrayCreator<char[]>() {
         @Override public char[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid char array length: " + len);
 
             switch (len) {
                 case 0:
@@ -185,7 +193,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<boolean[]> BOOLEAN_ARR_CREATOR = new ArrayCreator<boolean[]>() {
         @Override public boolean[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid boolean array length: " + len);
 
             switch (len) {
                 case 0:

From ebc4a1648a80fbbd485e4c351fce9bee163318f9 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 16 May 2017 11:30:29 +0300
Subject: [PATCH 163/516] DirectByteBufferStreamImpl: converted asserts into
 exceptions.

(cherry picked from commit 560ef60)
---
 .../v2/DirectByteBufferStreamImplV2.java      | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
index d7dc990848094..af45263e36a8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
@@ -1,4 +1,4 @@
-/*
+ /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -27,6 +27,7 @@
 import java.util.Map;
 import java.util.RandomAccess;
 import java.util.UUID;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.direct.stream.DirectByteBufferStream;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -80,7 +81,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<byte[]> BYTE_ARR_CREATOR = new ArrayCreator<byte[]>() {
         @Override public byte[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid byte array length: " + len);
 
             switch (len) {
                 case 0:
@@ -95,7 +97,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<short[]> SHORT_ARR_CREATOR = new ArrayCreator<short[]>() {
         @Override public short[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid short array length: " + len);
 
             switch (len) {
                 case 0:
@@ -110,7 +113,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<int[]> INT_ARR_CREATOR = new ArrayCreator<int[]>() {
         @Override public int[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid int array length: " + len);
 
             switch (len) {
                 case 0:
@@ -125,7 +129,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<long[]> LONG_ARR_CREATOR = new ArrayCreator<long[]>() {
         @Override public long[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid long array length: " + len);
 
             switch (len) {
                 case 0:
@@ -140,7 +145,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<float[]> FLOAT_ARR_CREATOR = new ArrayCreator<float[]>() {
         @Override public float[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid float array length: " + len);
 
             switch (len) {
                 case 0:
@@ -155,7 +161,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<double[]> DOUBLE_ARR_CREATOR = new ArrayCreator<double[]>() {
         @Override public double[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid double array length: " + len);
 
             switch (len) {
                 case 0:
@@ -170,7 +177,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<char[]> CHAR_ARR_CREATOR = new ArrayCreator<char[]>() {
         @Override public char[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid char array length: " + len);
 
             switch (len) {
                 case 0:
@@ -185,7 +193,8 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     /** */
     private static final ArrayCreator<boolean[]> BOOLEAN_ARR_CREATOR = new ArrayCreator<boolean[]>() {
         @Override public boolean[] create(int len) {
-            assert len >= 0;
+            if (len < 0)
+                throw new IgniteException("Read invalid boolean array length: " + len);
 
             switch (len) {
                 case 0:

From 9cd7e0f8d132f9b7c496fe64f75f271ef60da5eb Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Thu, 9 Feb 2017 16:44:41 +0700
Subject: [PATCH 164/516] IGNITE-4676 Fixed hang if closure executed nested
 internal task with continuation. Added test. (cherry picked from commit
 e7a5307)

---
 .../processors/job/GridJobWorker.java         |  4 +
 .../internal/GridContinuousTaskSelfTest.java  | 79 +++++++++++++++++++
 2 files changed, 83 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index 6a00d96a0abc2..acefde72eefab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -617,6 +617,10 @@ else if (X.hasCause(e, GridServiceNotFoundException.class) ||
                 // Finish here only if not held by this thread.
                 if (!HOLD.get())
                     finishJob(res, ex, sndRes);
+                else
+                    // Make sure flag is not set for current thread.
+                    // This may happen in case of nested internal task call with continuation.
+                    HOLD.set(false);
 
                 ctx.job().currentTaskSession(null);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
index 98e3c5afc8e2d..cec288714c100 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridContinuousTaskSelfTest.java
@@ -21,10 +21,12 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.concurrent.Callable;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
@@ -43,7 +45,9 @@
 import org.apache.ignite.compute.ComputeTaskSessionAttributeListener;
 import org.apache.ignite.compute.ComputeTaskSessionFullSupport;
 import org.apache.ignite.compute.ComputeTaskSplitAdapter;
+import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
 import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.resources.TaskContinuousMapperResource;
@@ -51,6 +55,7 @@
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Continuous task test.
@@ -195,6 +200,80 @@ public void testMultipleHoldccCalls() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If test failed.
+     */
+    public void testClosureWithNestedInternalTask() throws Exception {
+        try {
+            IgniteEx ignite = startGrid(0);
+
+            ComputeTaskInternalFuture<String> fut = ignite.context().closure().callAsync(GridClosureCallMode.BALANCE, new Callable<String>() {
+                /** */
+                @IgniteInstanceResource
+                private IgniteEx g;
+
+                @Override public String call() throws Exception {
+                    return g.compute(g.cluster()).execute(NestedHoldccTask.class, null);
+                }
+            }, ignite.cluster().nodes());
+
+            assertEquals("DONE", fut.get(3000));
+        }
+        finally {
+            stopGrid(0, true);
+        }
+    }
+
+    /** Test task with continuation. */
+    @GridInternal
+    public static class NestedHoldccTask extends ComputeTaskAdapter<String, String> {
+        /** {@inheritDoc} */
+        @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
+            @Nullable String arg) throws IgniteException {
+            Map<ComputeJob, ClusterNode> map = new HashMap<>();
+
+            for (ClusterNode node : subgrid)
+                map.put(new NestedHoldccJob(), node);
+
+            return map;
+
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public String reduce(List<ComputeJobResult> results) throws IgniteException {
+            return results.get(0).getData();
+        }
+    }
+
+    /** Test job. */
+    public static class NestedHoldccJob extends ComputeJobAdapter {
+        /** */
+        @JobContextResource
+        private ComputeJobContext jobCtx;
+
+        /** */
+        private int cnt = 0;
+
+        /** {@inheritDoc} */
+        @Override public Object execute() throws IgniteException {
+            if (cnt < 1) {
+                cnt++;
+
+                jobCtx.holdcc();
+
+                new Timer().schedule(new TimerTask() {
+                    @Override public void run() {
+                        jobCtx.callcc();
+                    }
+                }, 500);
+
+                return "NOT DONE";
+            }
+
+            return "DONE";
+        }
+    }
+
     /** */
     @SuppressWarnings({"PublicInnerClass"})
     public static class TestMultipleHoldccCallsClosure implements IgniteClosure<Object, Boolean> {

From 43bcc15127bd3fd7ac4e277da6da9e5fb6a855c0 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Thu, 30 Mar 2017 11:08:10 +0700
Subject: [PATCH 165/516] IGNITE-4838 Fixed internal task detection logic.
 Added tests. (cherry picked from commit ba68c6c)

---
 .../processors/task/GridTaskProcessor.java    |  9 ++++-
 .../internal/GridTaskExecutionSelfTest.java   | 34 +++++++++++++++++++
 ...cutionWithoutPeerClassLoadingSelfTest.java | 31 +++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |  2 ++
 4 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index d32b51c0867ca..935686456e593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -608,6 +608,13 @@ else if (task != null) {
         if (subjId == null)
             subjId = ctx.localNodeId();
 
+        boolean internal = false;
+
+        if (dep == null || taskCls == null)
+            assert deployEx != null;
+        else
+            internal = dep.internalTask(task, taskCls);
+
         // Creates task session with task name and task version.
         GridTaskSessionImpl ses = ctx.session().createTaskSession(
             sesId,
@@ -621,7 +628,7 @@ else if (task != null) {
             Collections.<ComputeJobSibling>emptyList(),
             Collections.emptyMap(),
             fullSup,
-            dep != null && dep.internalTask(task, taskCls),
+            internal,
             subjId);
 
         ComputeTaskInternalFuture<R> fut = new ComputeTaskInternalFuture<>(ses, ctx);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
index 996acd686f3cd..cc6a1eae6b2c6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
@@ -22,8 +22,10 @@
 import org.apache.ignite.GridTestTask;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteDeploymentException;
 import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
@@ -45,6 +47,20 @@ public GridTaskExecutionSelfTest() {
         super(false);
     }
 
+    /** */
+    protected boolean peerClassLoadingEnabled() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setPeerClassLoadingEnabled(peerClassLoadingEnabled());
+
+        return cfg;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         startGrid(1);
@@ -144,4 +160,22 @@ public void testJobIdCollision() throws Exception {
         for (IgniteFuture<Object> fut : futs)
             fut.get();
     }
+
+    /**
+     * Test execution of non-existing task by name IGNITE-4838.
+     *
+     * @throws Exception If failed.
+     */
+    public void testExecuteTaskWithInvalidName() throws Exception {
+        try {
+            ComputeTaskFuture<?> fut = ignite.compute().execute("invalid.task.name", null);
+
+            fut.get();
+
+            assert false : "Should never be reached due to exception thrown.";
+        }
+        catch (IgniteDeploymentException e) {
+            info("Received correct exception: " + e);
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java
new file mode 100644
index 0000000000000..45e65cd9db017
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+
+/**
+ * Task execution test.
+ */
+@GridCommonTest(group = "Kernal Self")
+public class GridTaskExecutionWithoutPeerClassLoadingSelfTest extends GridTaskExecutionSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean peerClassLoadingEnabled() {
+        return false;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 8a501fdb801d6..ae64c95fb30b6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -51,6 +51,7 @@
 import org.apache.ignite.internal.GridStopWithWaitSelfTest;
 import org.apache.ignite.internal.GridTaskCancelSingleNodeSelfTest;
 import org.apache.ignite.internal.GridTaskExecutionSelfTest;
+import org.apache.ignite.internal.GridTaskExecutionWithoutPeerClassLoadingSelfTest;
 import org.apache.ignite.internal.GridTaskFailoverAffinityRunTest;
 import org.apache.ignite.internal.GridTaskFailoverSelfTest;
 import org.apache.ignite.internal.GridTaskFutureImplStopGridSelfTest;
@@ -100,6 +101,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCancelUnusedJobSelfTest.class);
         suite.addTestSuite(GridTaskJobRejectSelfTest.class);
         suite.addTestSuite(GridTaskExecutionSelfTest.class);
+        suite.addTestSuite(GridTaskExecutionWithoutPeerClassLoadingSelfTest.class);
         suite.addTestSuite(GridFailoverSelfTest.class);
         suite.addTestSuite(GridTaskListenerSelfTest.class);
         suite.addTestSuite(GridFailoverTopologySelfTest.class);

From 72882126c047b937bd5ed93c85e28262530e4977 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Thu, 30 Mar 2017 11:08:10 +0700
Subject: [PATCH 166/516] IGNITE-4838 Fixed internal task detection logic.
 Added tests. (cherry picked from commit ba68c6c)

---
 .../processors/task/GridTaskProcessor.java    |  9 ++++-
 .../internal/GridTaskExecutionSelfTest.java   | 34 +++++++++++++++++++
 ...cutionWithoutPeerClassLoadingSelfTest.java | 31 +++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |  2 ++
 4 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index d32b51c0867ca..935686456e593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -608,6 +608,13 @@ else if (task != null) {
         if (subjId == null)
             subjId = ctx.localNodeId();
 
+        boolean internal = false;
+
+        if (dep == null || taskCls == null)
+            assert deployEx != null;
+        else
+            internal = dep.internalTask(task, taskCls);
+
         // Creates task session with task name and task version.
         GridTaskSessionImpl ses = ctx.session().createTaskSession(
             sesId,
@@ -621,7 +628,7 @@ else if (task != null) {
             Collections.<ComputeJobSibling>emptyList(),
             Collections.emptyMap(),
             fullSup,
-            dep != null && dep.internalTask(task, taskCls),
+            internal,
             subjId);
 
         ComputeTaskInternalFuture<R> fut = new ComputeTaskInternalFuture<>(ses, ctx);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
index 996acd686f3cd..cc6a1eae6b2c6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionSelfTest.java
@@ -22,8 +22,10 @@
 import org.apache.ignite.GridTestTask;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteDeploymentException;
 import org.apache.ignite.compute.ComputeJobContext;
 import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
@@ -45,6 +47,20 @@ public GridTaskExecutionSelfTest() {
         super(false);
     }
 
+    /** */
+    protected boolean peerClassLoadingEnabled() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setPeerClassLoadingEnabled(peerClassLoadingEnabled());
+
+        return cfg;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         startGrid(1);
@@ -144,4 +160,22 @@ public void testJobIdCollision() throws Exception {
         for (IgniteFuture<Object> fut : futs)
             fut.get();
     }
+
+    /**
+     * Test execution of non-existing task by name IGNITE-4838.
+     *
+     * @throws Exception If failed.
+     */
+    public void testExecuteTaskWithInvalidName() throws Exception {
+        try {
+            ComputeTaskFuture<?> fut = ignite.compute().execute("invalid.task.name", null);
+
+            fut.get();
+
+            assert false : "Should never be reached due to exception thrown.";
+        }
+        catch (IgniteDeploymentException e) {
+            info("Received correct exception: " + e);
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java
new file mode 100644
index 0000000000000..45e65cd9db017
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridTaskExecutionWithoutPeerClassLoadingSelfTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+
+/**
+ * Task execution test.
+ */
+@GridCommonTest(group = "Kernal Self")
+public class GridTaskExecutionWithoutPeerClassLoadingSelfTest extends GridTaskExecutionSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean peerClassLoadingEnabled() {
+        return false;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 8a501fdb801d6..ae64c95fb30b6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -51,6 +51,7 @@
 import org.apache.ignite.internal.GridStopWithWaitSelfTest;
 import org.apache.ignite.internal.GridTaskCancelSingleNodeSelfTest;
 import org.apache.ignite.internal.GridTaskExecutionSelfTest;
+import org.apache.ignite.internal.GridTaskExecutionWithoutPeerClassLoadingSelfTest;
 import org.apache.ignite.internal.GridTaskFailoverAffinityRunTest;
 import org.apache.ignite.internal.GridTaskFailoverSelfTest;
 import org.apache.ignite.internal.GridTaskFutureImplStopGridSelfTest;
@@ -100,6 +101,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCancelUnusedJobSelfTest.class);
         suite.addTestSuite(GridTaskJobRejectSelfTest.class);
         suite.addTestSuite(GridTaskExecutionSelfTest.class);
+        suite.addTestSuite(GridTaskExecutionWithoutPeerClassLoadingSelfTest.class);
         suite.addTestSuite(GridFailoverSelfTest.class);
         suite.addTestSuite(GridTaskListenerSelfTest.class);
         suite.addTestSuite(GridFailoverTopologySelfTest.class);

From 0d3d93cd4eeb589f1b6a11b48e429defad01c82f Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 18 May 2017 19:11:08 +0300
Subject: [PATCH 167/516] IGNITE-4842 Now containsKey() respects
 isReadFromBackup() flag.

(cherry picked from commit d84fd29)
---
 .../processors/cache/GridCacheAdapter.java    |   4 +-
 .../IgniteCacheContainsKeyAtomicTest.java     | 103 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   3 +
 3 files changed, 108 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheContainsKeyAtomicTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index a50661e137600..da4893229799f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -642,7 +642,7 @@ public void onKernalStop() {
 
         return (IgniteInternalFuture)getAsync(
             key,
-            /*force primary*/false,
+            /*force primary*/ !ctx.config().isReadFromBackup(),
             /*skip tx*/false,
             /*subj id*/null,
             /*task name*/null,
@@ -669,7 +669,7 @@ public void onKernalStop() {
 
         return getAllAsync(
             keys,
-            /*force primary*/false,
+            /*force primary*/ !ctx.config().isReadFromBackup(),
             /*skip tx*/false,
             /*subj id*/null,
             /*task name*/null,
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheContainsKeyAtomicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheContainsKeyAtomicTest.java
new file mode 100644
index 0000000000000..981d245245a20
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheContainsKeyAtomicTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * Verifies that containsKey() works as expected on atomic cache.
+ */
+public class IgniteCacheContainsKeyAtomicTest extends GridCacheAbstractSelfTest {
+    /** Cache name. */
+    public static final String CACHE_NAME = "replicated";
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 4;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        IgniteCache cache = ignite(0).cache(CACHE_NAME);
+
+        if (cache != null)
+            cache.clear();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testContainsPutIfAbsent() throws Exception {
+        checkPutIfAbsent(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testContainsPutIfAbsentAll() throws Exception {
+        checkPutIfAbsent(true);
+    }
+
+    /**
+     * @param all Check for set of keys.
+     * @throws Exception If failed.
+     */
+    private void checkPutIfAbsent(final boolean all) throws Exception {
+        Ignite srv = ignite(0);
+
+        final IgniteCache<Integer, Integer> cache1 = srv.getOrCreateCache(replicatedCache());
+        final IgniteCache<Integer, Integer> cache2 = ignite(1).getOrCreateCache(replicatedCache());
+
+        final AtomicInteger fails = new AtomicInteger(0);
+
+        GridTestUtils.runMultiThreaded(new Runnable() {
+            @Override public void run() {
+                for (int i = 0; i < 100; i++) {
+                    if (!cache1.putIfAbsent(i, i)) {
+                        if (all ? !cache2.containsKeys(Collections.singleton(i)) : !cache2.containsKey(i))
+                            fails.incrementAndGet();
+                    }
+                }
+            }
+        }, 100, "put-if-abs");
+
+        assertEquals(0, fails.get());
+    }
+
+    /**
+     * @return replicated cache configuration.
+     */
+    private CacheConfiguration<Integer, Integer> replicatedCache() {
+        return new CacheConfiguration<Integer, Integer>(CACHE_NAME)
+            .setAtomicityMode(ATOMIC)
+            .setWriteSynchronizationMode(FULL_SYNC)
+            .setReadFromBackup(false) // containsKey() must respect this flag
+            .setCacheMode(REPLICATED);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 914aedb40764a..a13a9a4d7245a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicStoreValueTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheConfigurationDefaultTemplateTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheConfigurationTemplateTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheContainsKeyAtomicTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheDynamicStopSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheGetCustomCollectionsSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheInvokeReadThroughSingleNodeTest;
@@ -339,6 +340,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class);
 
+        suite.addTestSuite(IgniteCacheContainsKeyAtomicTest.class);
+
         return suite;
     }
 }
\ No newline at end of file

From 2a818d36395dd1af23acf444adf396b2e2edbede Mon Sep 17 00:00:00 2001
From: Konstantin Dudkov <kdudkov@ya.ru>
Date: Mon, 22 May 2017 16:28:07 +0300
Subject: [PATCH 168/516] Fixed "IGNITE-4205 CassandraCacheStore should start
 IiteThread threads in loadCache() method"

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../store/cassandra/CassandraCacheStore.java  | 13 ++-
 .../tests/IgnitePersistentStoreTest.java      | 62 +++++++++++++
 .../loadall_blob/ignite-config.xml            | 90 +++++++++++++++++++
 .../loadall_blob/persistence-settings.xml     | 29 ++++++
 .../store/jdbc/CacheAbstractJdbcStore.java    |  6 +-
 5 files changed, 198 insertions(+), 2 deletions(-)
 create mode 100644 modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml
 create mode 100644 modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/persistence-settings.xml

diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
index 2e1d3ea70318d..dabf1b09a588e 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
@@ -31,6 +31,7 @@
 import javax.cache.Cache;
 import javax.cache.integration.CacheLoaderException;
 import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.store.CacheStore;
@@ -47,7 +48,9 @@
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.logger.NullLogger;
 import org.apache.ignite.resources.CacheStoreSessionResource;
+import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.thread.IgniteThreadFactory;
 
 /**
  * Implementation of {@link CacheStore} backed by Cassandra database.
@@ -59,6 +62,14 @@ public class CassandraCacheStore<K, V> implements CacheStore<K, V> {
     /** Connection attribute property name. */
     private static final String ATTR_CONN_PROP = "CASSANDRA_STORE_CONNECTION";
 
+    /** Thread name. */
+    private static final String CACHE_LOADER_THREAD_NAME = "cassandra-cache-loader";
+
+    /** Auto-injected ignite instance. */
+    @SuppressWarnings("unused")
+    @IgniteInstanceResource
+    private Ignite ignite;
+
     /** Auto-injected store session. */
     @CacheStoreSessionResource
     private CacheStoreSession storeSes;
@@ -99,7 +110,7 @@ public CassandraCacheStore(DataSource dataSrc, KeyValuePersistenceSettings setti
         Collection<Future<?>> futs = new ArrayList<>(args.length);
 
         try {
-            pool = Executors.newFixedThreadPool(maxPoolSize);
+            pool = Executors.newFixedThreadPool(maxPoolSize, new IgniteThreadFactory(ignite.name(), CACHE_LOADER_THREAD_NAME));
 
             CassandraSession ses = getCassandraSession();
 
diff --git a/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java b/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
index 51d08855388c6..75dff66de8f84 100644
--- a/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
+++ b/modules/cassandra/src/test/java/org/apache/ignite/tests/IgnitePersistentStoreTest.java
@@ -23,9 +23,11 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
+import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.tests.pojos.Person;
 import org.apache.ignite.tests.pojos.PersonId;
@@ -237,6 +239,35 @@ public void blobStrategyTest() {
         }
     }
 
+
+    /** */
+    @Test
+    public void blobBinaryLoadCacheTest() {
+        Ignition.stopAll(true);
+
+        try (Ignite ignite = Ignition.start("org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml")) {
+            IgniteCache<Long, PojoPerson> personCache = ignite.getOrCreateCache("cache2");
+
+            assert ignite.configuration().getMarshaller() instanceof BinaryMarshaller;
+
+            personCache.put(1L, new PojoPerson(1, "name"));
+
+            assert personCache.withKeepBinary().get(1L) instanceof BinaryObject;
+        }
+
+        Ignition.stopAll(true);
+
+        try (Ignite ignite = Ignition.start("org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml")) {
+            IgniteCache<Long, PojoPerson> personCache = ignite.getOrCreateCache("cache2");
+
+            personCache.loadCache(null, null);
+
+            PojoPerson person = personCache.get(1L);
+
+            LOGGER.info("loadCache tests passed");
+        }
+    }
+
     /** */
     @Test
     public void pojoStrategyTest() {
@@ -377,4 +408,35 @@ public void loadCacheTest() {
 
         LOGGER.info("loadCache test passed");
     }
+
+
+    /** */
+    public static class PojoPerson {
+        /** */
+        private int id;
+
+        /** */
+        private String name;
+
+        /** */
+        public PojoPerson() {
+            // No-op.
+        }
+
+        /** */
+        public PojoPerson(int id, String name) {
+            this.id = id;
+            this.name = name;
+        }
+
+        /** */
+        public int getId() {
+            return id;
+        }
+
+        /** */
+        public String getName() {
+            return name;
+        }
+    }
 }
diff --git a/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml b/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml
new file mode 100644
index 0000000000000..115e263e017d8
--- /dev/null
+++ b/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/ignite-config.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~      http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+    <!-- Cassandra connection settings -->
+    <import resource="classpath:org/apache/ignite/tests/cassandra/connection-settings.xml"/>
+
+    <!-- Persistence settings for 'cache2' -->
+    <bean id="cache2_persistence_settings"
+          class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings">
+        <constructor-arg type="org.springframework.core.io.Resource"
+                         value="classpath:org/apache/ignite/tests/persistence/loadall_blob/persistence-settings.xml"/>
+    </bean>
+
+    <!-- Ignite configuration -->
+    <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+
+        <property name="marshaller">
+            <bean class="org.apache.ignite.internal.binary.BinaryMarshaller"/>
+        </property>
+
+        <property name="binaryConfiguration">
+            <bean class="org.apache.ignite.configuration.BinaryConfiguration">
+                <property name="compactFooter" value="false"/>
+            </bean>
+        </property>
+
+        <property name="cacheConfiguration">
+            <list>
+                <!-- Configuring persistence for "cache2" cache -->
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="name" value="cache2"/>
+                    <property name="readThrough" value="true"/>
+                    <property name="writeThrough" value="true"/>
+                    <property name="storeKeepBinary" value="true"/>
+                    <property name="cacheStoreFactory">
+                        <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory">
+                            <property name="dataSourceBean" value="cassandraAdminDataSource"/>
+                            <property name="persistenceSettingsBean" value="cache2_persistence_settings"/>
+                        </bean>
+                    </property>
+                </bean>
+            </list>
+        </property>
+
+        <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. -->
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <!--
+                        Ignite provides several options for automatic discovery that can be used
+                        instead os static IP based discovery. For information on all options refer
+                        to our documentation: http://apacheignite.readme.io/docs/cluster-config
+                    -->
+                    <!-- Uncomment static IP finder to enable static-based discovery of initial nodes. -->
+                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+                        <!--<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">-->
+                        <property name="addresses">
+                            <list>
+                                <!-- In distributed environment, replace with actual host IP address. -->
+                                <value>127.0.0.1:47500..47509</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+    </bean>
+</beans>
diff --git a/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/persistence-settings.xml b/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/persistence-settings.xml
new file mode 100644
index 0000000000000..e872201fcc180
--- /dev/null
+++ b/modules/cassandra/src/test/resources/org/apache/ignite/tests/persistence/loadall_blob/persistence-settings.xml
@@ -0,0 +1,29 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<persistence keyspace="test1" table="blob_test3">
+    <!-- By default Java standard serialization is going to be used -->
+    <keyPersistence class="java.lang.Long"
+                    strategy="BLOB"
+                    column="key"/>
+
+    <!-- Kryo serialization specified to be used -->
+    <valuePersistence class="org.apache.ignite.tests.pojos.Person"
+                      strategy="BLOB"
+                      serializer="org.apache.ignite.cache.store.cassandra.serializer.JavaSerializer"
+                      column="value"/>
+</persistence>
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
index e211fadf2fb74..817b1a5da8531 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
@@ -72,6 +72,7 @@
 import org.apache.ignite.resources.CacheStoreSessionResource;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.thread.IgniteThreadFactory;
 import org.apache.ignite.transactions.Transaction;
 import org.jetbrains.annotations.Nullable;
 
@@ -121,6 +122,9 @@
  * </pre>
  */
 public abstract class CacheAbstractJdbcStore<K, V> implements CacheStore<K, V>, LifecycleAware {
+    /** Thread name. */
+    private static final String CACHE_LOADER_THREAD_NAME = "jdbc-cache-loader";
+
     /** Connection attribute property name. */
     protected static final String ATTR_CONN_PROP = "JDBC_STORE_CONNECTION";
 
@@ -730,7 +734,7 @@ protected Integer columnIndex(Map<String, Integer> loadColIdxs, String dbName) {
         String cacheName = session().cacheName();
 
         try {
-            pool = Executors.newFixedThreadPool(maxPoolSize);
+            pool = Executors.newFixedThreadPool(maxPoolSize, new IgniteThreadFactory(ignite.name(), CACHE_LOADER_THREAD_NAME));
 
             Collection<Future<?>> futs = new ArrayList<>();
 

From 04fadd4a499239176ba21c390d93e30809abb4c1 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 23 May 2017 15:42:20 +0300
Subject: [PATCH 169/516] IGNITE-5223 Allow use local binary metadata cache if
 it's possible

---
 .../apache/ignite/IgniteSystemProperties.java |   8 +
 .../ignite/internal/binary/BinaryContext.java |   4 +
 .../internal/binary/BinaryReaderExImpl.java   |  23 +-
 .../CacheObjectBinaryProcessorImpl.java       |  98 ++++--
 ...inaryMarshallerLocalMetadataCacheTest.java | 297 ++++++++++++++++++
 .../IgniteBinaryObjectsTestSuite.java         |   3 +
 6 files changed, 406 insertions(+), 27 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCacheTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index d77b2fb2838d4..713defeda552c 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -515,6 +515,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI = "IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI";
 
+    /**
+     * Use local metadata cache instead of distributed one. May be used only when binary objects schema
+     * are not modified and all classes available on each node. Classes that implements Binarylizable are
+     * not supported.
+     * @deprecated Should be removed in Apache Ignite 2.0.
+     */
+    public static final String IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE = "IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 7b21dfbed51f0..9f66b3ed50046 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -612,6 +612,7 @@ else if (cpElement.isFile()) {
 
     /**
      * @param cls Class.
+     * @param deserialize If {@code false}, metadata will be updated.
      * @return Class descriptor.
      * @throws BinaryObjectException In case of error.
      */
@@ -664,6 +665,7 @@ else if (!desc.registered()) {
      * @param userType User type or not.
      * @param typeId Type ID.
      * @param ldr Class loader.
+     * @param deserialize If {@code false}, metadata will be updated.
      * @return Class descriptor.
      */
     public BinaryClassDescriptor descriptorForTypeId(
@@ -719,6 +721,7 @@ public BinaryClassDescriptor descriptorForTypeId(
      * Creates and registers {@link BinaryClassDescriptor} for the given {@code class}.
      *
      * @param cls Class.
+     * @param deserialize If {@code false}, metadata will be updated.
      * @return Class descriptor.
      */
     private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean deserialize) {
@@ -759,6 +762,7 @@ private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean dese
      * Creates and registers {@link BinaryClassDescriptor} for the given user {@code class}.
      *
      * @param cls Class.
+     * @param deserialize If {@code false}, metadata will be updated.
      * @return Class descriptor.
      */
     private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean deserialize) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
index 775f237787cfc..ad2e736fa5f34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
@@ -26,6 +26,7 @@
 import java.util.Date;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryCollectionFactory;
 import org.apache.ignite.binary.BinaryInvalidTypeException;
 import org.apache.ignite.binary.BinaryMapFactory;
@@ -1722,15 +1723,23 @@ public BinarySchema getOrCreateSchema() {
             if (fieldIdLen != BinaryUtils.FIELD_ID_LEN) {
                 BinaryTypeImpl type = (BinaryTypeImpl)ctx.metadata(typeId);
 
-                if (type == null || type.metadata() == null)
-                    throw new BinaryObjectException("Cannot find metadata for object with compact footer: " +
-                        typeId);
+                if (type == null || type.metadata() == null || type.metadata().schemas().isEmpty()) {
+                    if (IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE, false)) {
+                        BinaryClassDescriptor desc = ctx.descriptorForTypeId(true, typeId, getClass().getClassLoader(), false);
 
-                for (BinarySchema typeSchema : type.metadata().schemas()) {
-                    if (schemaId == typeSchema.schemaId()) {
-                        schema = typeSchema;
+                        schema = desc.schema();
+                    }
+                    else
+                        throw new BinaryObjectException("Cannot find metadata for object with compact footer: " +
+                            typeId);
+                }
+                else {
+                    for (BinarySchema typeSchema : type.metadata().schemas()) {
+                        if (schemaId == typeSchema.schemaId()) {
+                            schema = typeSchema;
 
-                        break;
+                            break;
+                        }
                     }
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 1d60c422a6417..6b691c2236e52 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.IgniteBinary;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryBasicNameMapper;
 import org.apache.ignite.binary.BinaryField;
 import org.apache.ignite.binary.BinaryObject;
@@ -126,6 +127,9 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
     /** */
     private volatile IgniteCacheProxy<BinaryMetadataKey, BinaryMetadata> metaDataCache;
 
+    /** */
+    private final ConcurrentHashMap8<BinaryMetadataKey, BinaryMetadata> locMetadataCache;
+
     /** */
     private final ConcurrentHashMap8<Integer, BinaryTypeImpl> clientMetaDataCache;
 
@@ -175,6 +179,10 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
         clientNode = this.ctx.clientNode();
 
         clientMetaDataCache = clientNode ? new ConcurrentHashMap8<Integer, BinaryTypeImpl>() : null;
+
+        boolean useLocCache = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE, false);
+
+        locMetadataCache = useLocCache ? new ConcurrentHashMap8<BinaryMetadataKey, BinaryMetadata>() : null;
     }
 
     /** {@inheritDoc} */
@@ -190,7 +198,7 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
 
                     BinaryMetadata newMeta0 = ((BinaryTypeImpl)newMeta).metadata();
 
-                    if (metaDataCache == null) {
+                    if (metaDataCache == null && locMetadataCache == null) {
                         BinaryMetadata oldMeta = metaBuf.get(typeId);
                         BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0);
 
@@ -213,13 +221,13 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
                             return;
                     }
 
-                    assert metaDataCache != null;
+                    assert metaDataCache != null || locMetadataCache != null;
 
                     CacheObjectBinaryProcessorImpl.this.addMeta(typeId, newMeta0.wrap(binaryCtx));
                 }
 
                 @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
-                    if (metaDataCache == null)
+                    if (metaDataCache == null && locMetadataCache == null)
                         U.awaitQuiet(startLatch);
 
                     return CacheObjectBinaryProcessorImpl.this.metadata(typeId);
@@ -289,6 +297,12 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public void onUtilityCacheStarted() throws IgniteCheckedException {
+        if (locMetadataCache != null) {
+            startLatch.countDown();
+
+            return;
+        }
+
         IgniteCacheProxy<Object, Object> proxy = ctx.cache().jcache(CU.UTILITY_CACHE_NAME);
 
         boolean old = proxy.context().deploy().ignoreOwnership(true);
@@ -573,22 +587,31 @@ public GridBinaryMarshaller marshaller() {
 
         final BinaryMetadataKey key = new BinaryMetadataKey(typeId);
 
-        try {
-            BinaryMetadata oldMeta = metaDataCache.localPeek(key);
-            BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0);
+        if (locMetadataCache != null) {
+            locMetadataCache.merge(key, newMeta0, new ConcurrentHashMap8.BiFun<BinaryMetadata, BinaryMetadata, BinaryMetadata>() {
+                @Override public BinaryMetadata apply(BinaryMetadata curMeta, BinaryMetadata newMeta) {
+                    return BinaryUtils.mergeMetadata(curMeta, newMeta);
+                }
+            });
+        }
+        else {
+            try {
+                BinaryMetadata oldMeta = metaDataCache.localPeek(key);
+                BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0);
 
-            AffinityTopologyVersion topVer = ctx.cache().context().lockedTopologyVersion(null);
+                AffinityTopologyVersion topVer = ctx.cache().context().lockedTopologyVersion(null);
 
-            if (topVer == null)
-                topVer = ctx.cache().context().exchange().readyAffinityVersion();
+                if (topVer == null)
+                    topVer = ctx.cache().context().exchange().readyAffinityVersion();
 
-            BinaryObjectException err = metaDataCache.invoke(topVer, key, new MetadataProcessor(mergedMeta));
+                BinaryObjectException err = metaDataCache.invoke(topVer, key, new MetadataProcessor(mergedMeta));
 
-            if (err != null)
-                throw err;
-        }
-        catch (CacheException e) {
-            throw new BinaryObjectException("Failed to update meta data for type: " + newMeta.typeName(), e);
+                if (err != null)
+                    throw err;
+            }
+            catch (CacheException e) {
+                throw new BinaryObjectException("Failed to update meta data for type: " + newMeta.typeName(), e);
+            }
         }
     }
 
@@ -601,17 +624,28 @@ public GridBinaryMarshaller marshaller() {
                 if (typeMeta != null)
                     return typeMeta;
 
-                BinaryMetadata meta = metaDataCache.getTopologySafe(new BinaryMetadataKey(typeId));
+                BinaryMetadata meta;
+
+                if (locMetadataCache != null)
+                    meta = locMetadataCache.get(new BinaryMetadataKey(typeId));
+                else
+                    meta = metaDataCache.getTopologySafe(new BinaryMetadataKey(typeId));
 
                 return meta != null ? meta.wrap(binaryCtx) : null;
             }
             else {
                 BinaryMetadataKey key = new BinaryMetadataKey(typeId);
 
-                BinaryMetadata meta = metaDataCache.localPeek(key);
+                BinaryMetadata meta;
+
+                if (locMetadataCache != null)
+                    meta = locMetadataCache.get(key);
+                else {
+                    meta = metaDataCache.localPeek(key);
 
-                if (meta == null && !metaDataCache.context().preloader().syncFuture().isDone())
-                    meta = metaDataCache.getTopologySafe(key);
+                    if (meta == null && !metaDataCache.context().preloader().syncFuture().isDone())
+                        meta = metaDataCache.getTopologySafe(key);
+                }
 
                 return meta != null ? meta.wrap(binaryCtx) : null;
             }
@@ -630,7 +664,20 @@ public GridBinaryMarshaller marshaller() {
             for (Integer typeId : typeIds)
                 keys.add(new BinaryMetadataKey(typeId));
 
-            Map<BinaryMetadataKey, BinaryMetadata> meta = metaDataCache.getAll(keys);
+            Map<BinaryMetadataKey, BinaryMetadata> meta;
+
+            if (locMetadataCache != null) {
+                meta = new HashMap<>();
+
+                for (BinaryMetadataKey key : keys) {
+                    BinaryMetadata metadata = locMetadataCache.get(key);
+
+                    if (metadata != null)
+                        meta.put(key, metadata);
+                }
+            }
+            else
+                meta = metaDataCache.getAll(keys);
 
             Map<Integer, BinaryType> res = U.newHashMap(meta.size());
 
@@ -654,6 +701,17 @@ public GridBinaryMarshaller marshaller() {
                 }
             });
         else {
+            if (locMetadataCache != null) {
+                ConcurrentHashMap8.ValuesView<BinaryMetadataKey, BinaryMetadata> vals = locMetadataCache.values();
+
+                ArrayList<BinaryType> res = new ArrayList<>(vals.size());
+
+                for (BinaryMetadata metadata : vals)
+                    res.add(metadata.wrap(binaryCtx));
+
+                return res;
+            }
+
             return F.viewReadOnly(metaDataCache.entrySetx(metaPred),
                 new C1<Cache.Entry<BinaryMetadataKey, BinaryMetadata>, BinaryType>() {
                     private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCacheTest.java
new file mode 100644
index 0000000000000..ad3c5f3e0a83b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCacheTest.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.testframework.junits.IgniteTestResources;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE property.
+ */
+public class BinaryMarshallerLocalMetadataCacheTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName,
+        IgniteTestResources rsrcs) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName, rsrcs);
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        cfg.setClientMode(gridName.startsWith("client"));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE, "true");
+
+        startGrid(0);
+        startGrid(1);
+        startGrid("client");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        try {
+            stopAllGrids();
+        }
+        finally {
+            System.clearProperty(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testComputeLocalMetadata() throws Exception {
+        final BinaryObject obj = grid(0).binary().toBinary(new OptimizedContainer(new Optimized()));
+
+        ClusterGroup remotes = grid(0).cluster().forRemotes();
+
+        OptimizedContainer res = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
+            @Override public OptimizedContainer call() throws Exception {
+
+                return obj.deserialize();
+            }
+        });
+
+        OptimizedContainer res2 = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
+            @Override public OptimizedContainer call() throws Exception {
+
+                return obj.deserialize();
+            }
+        });
+
+        System.out.println(res);
+        System.out.println(res2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCacheLocalMetadata() throws Exception {
+        IgniteCache<Key, Object> cache = grid("client").createCache("cache");
+
+        Map<Key, Object> data = new HashMap<>();
+
+        for (int i = 0; i < 1000; i++)
+            data.put(new Key(i), new OptimizedContainer(new Optimized(String.valueOf(i))));
+
+        for (int i = 1000; i < 2000; i++)
+            data.put(new Key(i), new Simple(i, String.valueOf(i), new BigInteger(String.valueOf(i), 10)));
+
+        cache.putAll(data);
+
+        checkCache(cache, data);
+        checkCache(grid(0).<Key, Object>cache("cache"), data);
+        checkCache(grid(1).<Key, Object>cache("cache"), data);
+    }
+
+    /**
+     * @param cache Cache.
+     * @param data Data.
+     */
+    private void checkCache(IgniteCache<Key, Object> cache,
+        Map<Key, Object> data) {
+        for (Map.Entry<Key, Object> entry : cache.getAll(data.keySet()).entrySet())
+            assertEquals(data.get(entry.getKey()), entry.getValue());
+    }
+
+    /**
+     *
+     */
+    private static class Key {
+        /** */
+        private Integer i;
+
+        /**
+         * @param i I.
+         */
+        public Key(Integer i) {
+            this.i = i;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            Key key = (Key)o;
+
+            return i != null ? i.equals(key.i) : key.i == null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return i != null ? i.hashCode() : 0;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class OptimizedContainer {
+        /** */
+        private Optimized optim;
+
+        /**
+         * @param optim Val.
+         */
+        public OptimizedContainer(Optimized optim) {
+            this.optim = optim;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            OptimizedContainer container = (OptimizedContainer)o;
+
+            return optim != null ? optim.equals(container.optim) : container.optim == null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return optim != null ? optim.hashCode() : 0;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Optimized implements Externalizable {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private String fld;
+
+        /**
+         * @param fld Fld.
+         */
+        public Optimized(String fld) {
+            this.fld = fld;
+        }
+
+        /**
+         * Default constructor (required by Externalizable).
+         */
+        public Optimized() {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(ObjectOutput out) throws IOException {
+            U.writeUTFStringNullable(out, fld);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+            fld = U.readUTFStringNullable(in);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            Optimized optimized = (Optimized)o;
+
+            return fld != null ? fld.equals(optimized.fld) : optimized.fld == null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return fld != null ? fld.hashCode() : 0;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Simple {
+        /** I. */
+        private int i;
+
+        /** String. */
+        private String str;
+
+        /** Big integer. */
+        private BigInteger bigInteger;
+
+        /**
+         * @param i I.
+         * @param str String.
+         * @param bigInteger Big integer.
+         */
+        public Simple(int i, String str, BigInteger bigInteger) {
+            this.i = i;
+            this.str = str;
+            this.bigInteger = bigInteger;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            Simple simple = (Simple)o;
+
+            if (i != simple.i)
+                return false;
+
+            if (str != null ? !str.equals(simple.str) : simple.str != null)
+                return false;
+
+            return bigInteger != null ? bigInteger.equals(simple.bigInteger) : simple.bigInteger == null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            int res = i;
+
+            res = 31 * res + (str != null ? str.hashCode() : 0);
+            res = 31 * res + (bigInteger != null ? bigInteger.hashCode() : 0);
+
+            return res;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
index c1d9974837fb3..29e43e4593415 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.binary.BinaryFieldsOffheapSelfTest;
 import org.apache.ignite.internal.binary.BinaryFooterOffsetsHeapSelfTest;
 import org.apache.ignite.internal.binary.BinaryFooterOffsetsOffheapSelfTest;
+import org.apache.ignite.internal.binary.BinaryMarshallerLocalMetadataCacheTest;
 import org.apache.ignite.internal.binary.BinaryMarshallerSelfTest;
 import org.apache.ignite.internal.binary.BinaryObjectBuilderAdditionalSelfTest;
 import org.apache.ignite.internal.binary.BinaryObjectBuilderDefaultMappersSelfTest;
@@ -147,6 +148,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(GridCacheBinaryObjectUserClassloaderSelfTest.class);
 
+        suite.addTestSuite(BinaryMarshallerLocalMetadataCacheTest.class);
+
         return suite;
     }
 }

From b2040b7a95e421609bcf7ae05b56dc623310b409 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 23 May 2017 16:14:08 +0300
Subject: [PATCH 170/516] IGNITE-5259 Minor serialization fix

---
 modules/core/pom.xml                          |  3 +
 .../apache/ignite/IgniteSystemProperties.java | 12 ++-
 .../ignite/internal/IgniteNodeAttributes.java |  6 ++
 .../discovery/GridDiscoveryManager.java       | 39 ++++++++
 .../CacheContinuousQueryHandler.java          |  1 -
 .../top/GridTopologyCommandHandler.java       |  2 +
 .../processors/security/SecurityUtils.java    | 92 ++++++++++++++++++
 .../security/SecurityBasicPermissionSet.java  | 41 +++++++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 97 +++++++++++++++++--
 ...ridDiscoveryManagerAttributesSelfTest.java | 70 ++++++++++++-
 .../discovery/tcp/TestReconnectProcessor.java | 47 ++++++++-
 11 files changed, 390 insertions(+), 20 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java

diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 4c4343a9c41a1..4a14407ab131f 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -234,6 +234,9 @@
                     <exclude>**/*.java</exclude>
                 </excludes>
             </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+            </testResource>
         </testResources>
 
         <plugins>
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 713defeda552c..6827e0c248290 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -503,7 +503,7 @@ public final class IgniteSystemProperties {
     /**
      * Whether Ignite can access unaligned memory addresses.
      * <p>
-     * Defaults to {@code} false, meaning that unaligned access will be performed only on x86 architecture.
+     * Defaults to {@code false}, meaning that unaligned access will be performed only on x86 architecture.
      */
     public static final String IGNITE_UNALIGNED_MEMORY_ACCESS = "IGNITE_UNALIGNED_MEMORY_ACCESS";
 
@@ -523,6 +523,16 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE = "IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE";
 
+     /**
+     * When set to {@code true}, Ignite switches to compatibility mode with versions that don't
+     * support service security permissions. In this case security permissions will be ignored
+     * (if they set).
+     * <p>
+     *     Default is {@code false}, which means that service security permissions will be respected.
+     * </p>
+     */
+    public static final String IGNITE_SECURITY_COMPATIBILITY_MODE = "IGNITE_SECURITY_COMPATIBILITY_MODE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
index e7c984f22864e..436792459af4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
@@ -136,6 +136,9 @@ public final class IgniteNodeAttributes {
     /** Security subject for authenticated node. */
     public static final String ATTR_SECURITY_SUBJECT = ATTR_PREFIX + ".security.subject";
 
+    /** V2 security subject for authenticated node. */
+    public static final String ATTR_SECURITY_SUBJECT_V2 = ATTR_PREFIX + ".security.subject.v2";
+
     /** Client mode flag. */
     public static final String ATTR_CLIENT_MODE = ATTR_PREFIX + ".cache.client";
 
@@ -160,6 +163,9 @@ public final class IgniteNodeAttributes {
     /** Ignite services compatibility mode (can be {@code null}). */
     public static final String ATTR_SERVICES_COMPATIBILITY_MODE = ATTR_PREFIX + ".services.compatibility.enabled";
 
+    /** Ignite security compatibility mode. */
+    public static final String ATTR_SECURITY_COMPATIBILITY_MODE = ATTR_PREFIX + ".security.compatibility.enabled";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 2ec10705bedbb..b3ba83da68a8d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -121,6 +121,7 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCOVERY_HISTORY_SIZE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SECURITY_COMPATIBILITY_MODE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.IgniteSystemProperties.getInteger;
 import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
@@ -136,9 +137,12 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_COMPATIBILITY_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME;
 import static org.apache.ignite.internal.IgniteVersionUtils.VER;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.SERVICE_PERMISSIONS_SINCE;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.isSecurityCompatibilityMode;
 import static org.apache.ignite.plugin.segmentation.SegmentationPolicy.NOOP;
 
 /**
@@ -442,6 +446,9 @@ private void updateClientNodes(UUID leftNodeId) {
         spi.setMetricsProvider(createMetricsProvider());
 
         if (ctx.security().enabled()) {
+            if (isSecurityCompatibilityMode())
+                ctx.addNodeAttribute(ATTR_SECURITY_COMPATIBILITY_MODE, true);
+
             spi.setAuthenticator(new DiscoverySpiNodeAuthenticator() {
                 @Override public SecurityContext authenticateNode(ClusterNode node, SecurityCredentials cred) {
                     try {
@@ -1044,6 +1051,7 @@ private void checkAttributes(Iterable<ClusterNode> nodes) throws IgniteCheckedEx
         boolean locDelayAssign = locNode.attribute(ATTR_LATE_AFFINITY_ASSIGNMENT);
 
         Boolean locSrvcCompatibilityEnabled = locNode.attribute(ATTR_SERVICES_COMPATIBILITY_MODE);
+        Boolean locSecurityCompatibilityEnabled = locNode.attribute(ATTR_SECURITY_COMPATIBILITY_MODE);
 
         for (ClusterNode n : nodes) {
             int rmtJvmMajVer = nodeJavaMajorVersion(n);
@@ -1157,6 +1165,37 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                     ", rmtNodeAddrs=" + U.addressesAsString(n) +
                     ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
             }
+
+            if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) >= 0
+                && ctx.security().enabled() // Matters only if security enabled.
+               ) {
+                Boolean rmtSecurityCompatibilityEnabled = n.attribute(ATTR_SECURITY_COMPATIBILITY_MODE);
+
+                if (!F.eq(locSecurityCompatibilityEnabled, rmtSecurityCompatibilityEnabled)) {
+                    throw new IgniteCheckedException("Local node's " + IGNITE_SECURITY_COMPATIBILITY_MODE +
+                        " property value differs from remote node's value " +
+                        "(to make sure all nodes in topology have identical Ignite security compatibility mode enabled, " +
+                        "configure system property explicitly) " +
+                        "[locSecurityCompatibilityEnabled=" + locSecurityCompatibilityEnabled +
+                        ", rmtSecurityCompatibilityEnabled=" + rmtSecurityCompatibilityEnabled +
+                        ", locNodeAddrs=" + U.addressesAsString(locNode) +
+                        ", rmtNodeAddrs=" + U.addressesAsString(n) +
+                        ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
+                }
+            }
+
+            if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) < 0
+                && ctx.security().enabled() // Matters only if security enabled.
+                && (locSecurityCompatibilityEnabled == null || !locSecurityCompatibilityEnabled)) {
+                throw new IgniteCheckedException("Remote node does not support service security permissions. " +
+                    "To be able to join to it, local node must be started with " + IGNITE_SECURITY_COMPATIBILITY_MODE +
+                    " system property set to \"true\". " +
+                    "[locSecurityCompatibilityEnabled=" + locSecurityCompatibilityEnabled +
+                    ", locNodeAddrs=" + U.addressesAsString(locNode) +
+                    ", rmtNodeAddrs=" + U.addressesAsString(n) +
+                    ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ", " +
+                    ", rmtNodeVer" + n.version() + ']');
+            }
         }
 
         if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 5f00d580f4231..17f4308c0709b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -998,7 +998,6 @@ private static class PartitionRecovery {
 
             synchronized (pendingEvts) {
                 if (log.isDebugEnabled()) {
-
                     log.debug("Handling event [lastFiredEvt=" + lastFiredEvt +
                         ", curTop=" + curTop +
                         ", entUpdCnt=" + entry.updateCounter() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
index 297785ea760f2..3c68fbfdba92e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
@@ -58,6 +58,7 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_TCP_PORT;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_CONFIG;
 import static org.apache.ignite.internal.processors.rest.GridRestCommand.NODE;
 import static org.apache.ignite.internal.processors.rest.GridRestCommand.TOPOLOGY;
@@ -290,6 +291,7 @@ private GridClientNodeBean createNodeBean(ClusterNode node, boolean mtr, boolean
             attrs.remove(ATTR_CACHE);
             attrs.remove(ATTR_TX_CONFIG);
             attrs.remove(ATTR_SECURITY_SUBJECT);
+            attrs.remove(ATTR_SECURITY_SUBJECT_V2);
             attrs.remove(ATTR_SECURITY_CREDENTIALS);
 
             for (Iterator<Map.Entry<String, Object>> i = attrs.entrySet().iterator(); i.hasNext();) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
new file mode 100644
index 0000000000000..1016335888b3d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.security;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.plugin.security.SecurityPermission;
+
+/**
+ * Security utilities.
+ */
+public class SecurityUtils {
+    /** Version since service security supported. */
+    public static final IgniteProductVersion SERVICE_PERMISSIONS_SINCE = IgniteProductVersion.fromString("1.7.11");
+
+    /** Default serialization version. */
+    private final static int DFLT_SERIALIZE_VERSION = isSecurityCompatibilityMode() ? 1 : 2;
+
+    /** Current serialization version. */
+    private static final ThreadLocal<Integer> SERIALIZE_VERSION = new ThreadLocal<Integer>(){
+        @Override protected Integer initialValue() {
+            return DFLT_SERIALIZE_VERSION;
+        }
+    };
+
+    /**
+     * Private constructor.
+     */
+    private SecurityUtils() {
+    }
+
+    /**
+     * @return Security compatibility mode flag.
+     */
+    public static boolean isSecurityCompatibilityMode() {
+        return IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_SECURITY_COMPATIBILITY_MODE, false);
+    }
+
+    /**
+     * @param ver Serialize version.
+     */
+    public static void serializeVersion(int ver) {
+        SERIALIZE_VERSION.set(ver);
+    }
+
+    /**
+     * @return Serialize version.
+     */
+    public static int serializeVersion() {
+        return SERIALIZE_VERSION.get();
+    }
+
+    /**
+     * Sets default serialize version {@link #DFLT_SERIALIZE_VERSION}.
+     */
+    public static void restoreDefaultSerializeVersion() {
+        serializeVersion(DFLT_SERIALIZE_VERSION);
+    }
+
+    /**
+     * @return Allow all service permissions.
+     */
+    public static Map<String, Collection<SecurityPermission>> compatibleServicePermissions() {
+        Map<String, Collection<SecurityPermission>> srvcPerms = new HashMap<>();
+
+        srvcPerms.put("*", Arrays.asList(
+            SecurityPermission.SERVICE_CANCEL,
+            SecurityPermission.SERVICE_DEPLOY,
+            SecurityPermission.SERVICE_INVOKE));
+
+        return srvcPerms;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
index 44166d936c9aa..370eadd801f83 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
@@ -17,15 +17,24 @@
 
 package org.apache.ignite.plugin.security;
 
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.processors.security.SecurityUtils.compatibleServicePermissions;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.isSecurityCompatibilityMode;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.serializeVersion;
+
 /**
  * Simple implementation of {@link SecurityPermissionSet} interface. Provides
  * convenient way to specify permission set in the XML configuration.
@@ -44,7 +53,9 @@ public class SecurityBasicPermissionSet implements SecurityPermissionSet {
 
     /** Service permissions. */
     @GridToStringInclude
-    private Map<String, Collection<SecurityPermission>> servicePermissions = new HashMap<>();
+    private transient Map<String, Collection<SecurityPermission>> servicePermissions = isSecurityCompatibilityMode()
+            ? compatibleServicePermissions()
+            : new HashMap<String, Collection<SecurityPermission>>();
 
     /** System permissions. */
     @GridToStringInclude
@@ -158,6 +169,34 @@ public void setDefaultAllowAll(boolean dfltAllowAll) {
         return res;
     }
 
+    /**
+     * @param out Out.
+     */
+    private void writeObject(ObjectOutputStream out) throws IOException {
+        out.defaultWriteObject();
+
+        if (serializeVersion() >= 2)
+            U.writeMap(out, servicePermissions);
+    }
+
+    /**
+     * @param in In.
+     */
+    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+        in.defaultReadObject();
+
+        if (serializeVersion() >= 2)
+            servicePermissions = U.readMap(in);
+
+        if (servicePermissions == null) {
+            // Allow all for compatibility mode
+            if (serializeVersion() < 2)
+                servicePermissions = compatibleServicePermissions();
+            else
+                servicePermissions = Collections.emptyMap();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(SecurityBasicPermissionSet.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index afd1c2ba7fc66..58b362ffd0f15 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -72,6 +72,7 @@
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager;
 import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.internal.processors.security.SecurityUtils;
 import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.util.GridBoundedLinkedHashSet;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
@@ -947,7 +948,8 @@ private void localAuthentication(SecurityCredentials locCred){
 
             Map<String, Object> attrs = new HashMap<>(locNode.attributes());
 
-            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT, U.marshal(spi.marshaller(), subj));
+            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2, U.marshal(spi.marshaller(), subj));
+            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT, marshalWithSecurityVersion(subj, 1));
 
             locNode.setAttributes(attrs);
 
@@ -983,7 +985,16 @@ private boolean sendJoinRequestMessage() throws IgniteSpiException {
 
             for (InetSocketAddress addr : addrs) {
                 try {
-                    Integer res = sendMessageDirectly(joinReq, addr);
+                    Integer res;
+
+                    try {
+                        SecurityUtils.serializeVersion(1);
+
+                        res = sendMessageDirectly(joinReq, addr);
+                    }
+                    finally {
+                        SecurityUtils.restoreDefaultSerializeVersion();
+                    }
 
                     assert res != null;
 
@@ -1921,6 +1932,39 @@ private void processMessageFailedNodes(TcpDiscoveryAbstractMessage msg) {
         }
     }
 
+    /**
+     * @param obj Object.
+     * @param ver Security serialize version.
+     * @return Marshaled object.
+     */
+    private byte[] marshalWithSecurityVersion(Object obj, int ver) throws IgniteCheckedException {
+        try {
+            SecurityUtils.serializeVersion(ver);
+
+            return U.marshal(spi.marshaller(), obj);
+        }
+        finally {
+            SecurityUtils.restoreDefaultSerializeVersion();
+        }
+    }
+
+    /**
+     * @param bytes Marshaled object.
+     * @param ver Security serialize version.
+     * @return Unmarshaled object.
+     */
+    private <T> T unmarshalWithSecurityVersion(byte[] bytes, int ver) throws IgniteCheckedException {
+        try {
+            if (ver > 0)
+                SecurityUtils.serializeVersion(ver);
+
+            return spi.marshaller().unmarshal(bytes, U.resolveClassLoader(spi.ignite().configuration()));
+        }
+        finally {
+            SecurityUtils.restoreDefaultSerializeVersion();
+        }
+    }
+
     /**
      * Discovery messages history used for client reconnect.
      */
@@ -2995,6 +3039,8 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof
                                     pendingMsgs.customDiscardId);
 
                             try {
+                                SecurityUtils.serializeVersion(1);
+
                                 long tstamp = U.currentTimeMillis();
 
                                 if (timeoutHelper == null)
@@ -3033,6 +3079,8 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof
                                 }
                             }
                             finally {
+                                SecurityUtils.restoreDefaultSerializeVersion();
+
                                 clearNodeAddedMessage(msg);
                             }
 
@@ -3423,7 +3471,8 @@ else if (log.isDebugEnabled())
                             // Stick in authentication subject to node (use security-safe attributes for copy).
                             Map<String, Object> attrs = new HashMap<>(node.getAttributes());
 
-                            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT, U.marshal(spi.marshaller(), subj));
+                            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2, U.marshal(spi.marshaller(), subj));
+                            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT, marshalWithSecurityVersion(subj, 1));
 
                             node.setAttributes(attrs);
                         }
@@ -4073,9 +4122,22 @@ else if (!locNodeId.equals(node.id()) && ring.node(node.id()) != null) {
                         else {
                             SecurityContext subj = spi.nodeAuth.authenticateNode(node, cred);
 
-                            SecurityContext coordSubj = U.unmarshal(spi.marshaller(),
-                                node.<byte[]>attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT),
-                                U.resolveClassLoader(spi.ignite().configuration()));
+                            byte[] subjBytes = node.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT);
+                            byte[] subjBytesV2 = node.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2);
+
+                            SecurityContext coordSubj;
+
+                            try {
+                                if (subjBytesV2 == null)
+                                    SecurityUtils.serializeVersion(1);
+
+                                coordSubj = U.unmarshal(spi.marshaller(),
+                                    subjBytesV2 != null ? subjBytesV2 : subjBytes,
+                                    U.resolveClassLoader(spi.ignite().configuration()));
+                            }
+                            finally {
+                                SecurityUtils.restoreDefaultSerializeVersion();
+                            }
 
                             if (!permissionsEqual(coordSubj.subject().permissions(), subj.subject().permissions())) {
                                 // Node has not pass authentication.
@@ -4158,13 +4220,23 @@ else if (!locNodeId.equals(node.id()) && ring.node(node.id()) != null) {
                                     new TcpDiscoveryAuthFailedMessage(locNodeId, spi.locHost);
 
                                 try {
-                                    ClassLoader cl = U.resolveClassLoader(spi.ignite().configuration());
-
                                     byte[] rmSubj = node.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT);
                                     byte[] locSubj = locNode.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT);
 
-                                    SecurityContext rmCrd = spi.marshaller().unmarshal(rmSubj, cl);
-                                    SecurityContext locCrd = spi.marshaller().unmarshal(locSubj, cl);
+                                    byte[] rmSubjV2 = node.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2);
+                                    byte[] locSubjV2 = locNode.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2);
+
+                                    int ver = 1; // Compatible version.
+
+                                    if (rmSubjV2 != null && locSubjV2 != null) {
+                                        rmSubj = rmSubjV2;
+                                        locSubj = locSubjV2;
+
+                                        ver = 0; // Default version.
+                                    }
+
+                                    SecurityContext rmCrd = unmarshalWithSecurityVersion(rmSubj, ver);
+                                    SecurityContext locCrd = unmarshalWithSecurityVersion(locSubj, ver);
 
                                     if (!permissionsEqual(locCrd.subject().permissions(),
                                         rmCrd.subject().permissions())) {
@@ -5812,6 +5884,8 @@ else if (e.hasCause(ObjectStreamException.class) ||
 
                 while (!isInterrupted()) {
                     try {
+                        SecurityUtils.serializeVersion(1);
+
                         TcpDiscoveryAbstractMessage msg = U.unmarshal(spi.marshaller(), in,
                             U.resolveClassLoader(spi.ignite().configuration()));
 
@@ -6062,6 +6136,9 @@ else if (msg instanceof TcpDiscoveryLoopbackProblemMessage) {
 
                         return;
                     }
+                    finally {
+                        SecurityUtils.restoreDefaultSerializeVersion();
+                    }
                 }
             }
             finally {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
index ba8fa5b6e15a3..f0096db7bc8a6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
@@ -21,14 +21,18 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DeploymentMode;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TestReconnectPluginProvider;
+import org.apache.ignite.spi.discovery.tcp.TestReconnectProcessor;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SECURITY_COMPATIBILITY_MODE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS;
@@ -258,18 +262,69 @@ public void testServiceCompatibilityEnabled() throws Exception {
      * @throws Exception If failed.
      */
     private void doTestServiceCompatibilityEnabled(Object first, Object second, boolean fail) throws Exception {
+        doTestCompatibilityEnabled(IGNITE_SERVICES_COMPATIBILITY_MODE, first, second, fail);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSecurityCompatibilityEnabled() throws Exception {
+        TestReconnectPluginProvider.enabled = true;
+        TestReconnectProcessor.enabled = true;
+
+        try {
+            doTestSecurityCompatibilityEnabled(true, null, true);
+            doTestSecurityCompatibilityEnabled(true, false, true);
+            doTestSecurityCompatibilityEnabled(false, true, true);
+            doTestSecurityCompatibilityEnabled(null, true, true);
+
+            doTestSecurityCompatibilityEnabled(null, null, false);
+            doTestSecurityCompatibilityEnabled(null, false, false);
+            doTestSecurityCompatibilityEnabled(false, false, false);
+            doTestSecurityCompatibilityEnabled(false, null, false);
+            doTestSecurityCompatibilityEnabled(true, true, false);
+        }
+        finally {
+            TestReconnectPluginProvider.enabled = false;
+            TestReconnectProcessor.enabled = false;
+        }
+    }
+
+    /**
+     * @param first Service compatibility enabled flag for first node.
+     * @param second Service compatibility enabled flag for second node.
+     * @param fail Fail flag.
+     * @throws Exception If failed.
+     */
+    private void doTestSecurityCompatibilityEnabled(Object first, Object second, boolean fail) throws Exception {
+        doTestCompatibilityEnabled(IGNITE_SECURITY_COMPATIBILITY_MODE, first, second, fail);
+    }
+
+    /**
+     * @param prop System property.
+     * @param first Service compatibility enabled flag for first node.
+     * @param second Service compatibility enabled flag for second node.
+     * @param fail Fail flag.
+     * @throws Exception If failed.
+     */
+    private void doTestCompatibilityEnabled(String prop, Object first, Object second, boolean fail) throws Exception {
+        String backup = System.getProperty(prop);
         try {
             if (first != null)
-                System.setProperty(IGNITE_SERVICES_COMPATIBILITY_MODE, String.valueOf(first));
+                System.setProperty(prop, String.valueOf(first));
             else
-                System.clearProperty(IGNITE_SERVICES_COMPATIBILITY_MODE);
+                System.clearProperty(prop);
 
-            startGrid(0);
+            IgniteEx ignite = startGrid(0);
+
+            // Ignore if disabled security plugin used.
+            if (IGNITE_SECURITY_COMPATIBILITY_MODE.equals(prop) && !ignite.context().security().enabled())
+                return;
 
             if (second != null)
-                System.setProperty(IGNITE_SERVICES_COMPATIBILITY_MODE, String.valueOf(second));
+                System.setProperty(prop, String.valueOf(second));
             else
-                System.clearProperty(IGNITE_SERVICES_COMPATIBILITY_MODE);
+                System.clearProperty(prop);
 
             try {
                 startGrid(1);
@@ -284,6 +339,11 @@ private void doTestServiceCompatibilityEnabled(Object first, Object second, bool
         }
         finally {
             stopAllGrids();
+
+            if (backup != null)
+                System.setProperty(prop, backup);
+            else
+                System.clearProperty(prop);
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
index f0ed35c4ccfab..2476bd3d787bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TestReconnectProcessor.java
@@ -17,11 +17,13 @@
 
 package org.apache.ignite.spi.discovery.tcp;
 
+import java.io.Serializable;
 import java.util.Collection;
 import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
 import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
 import org.apache.ignite.internal.processors.security.SecurityContext;
@@ -37,6 +39,9 @@
  * Updates node attributes on disconnect.
  */
 public class TestReconnectProcessor extends GridProcessorAdapter implements GridSecurityProcessor {
+    /** Enabled flag. */
+    public static boolean enabled;
+
     /**
      * @param ctx Kernal context.
      */
@@ -44,10 +49,15 @@ protected TestReconnectProcessor(GridKernalContext ctx) {
         super(ctx);
     }
 
+    /** {@inheritDoc} */
+    @Override public void start() throws IgniteCheckedException {
+        ctx.addNodeAttribute(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS, new SecurityCredentials());
+    }
+
     /** {@inheritDoc} */
     @Override public SecurityContext authenticateNode(ClusterNode node,
         SecurityCredentials cred) throws IgniteCheckedException {
-        return null;
+        return new TestSecurityContext();
     }
 
     /** {@inheritDoc} */
@@ -83,11 +93,44 @@ protected TestReconnectProcessor(GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public boolean enabled() {
-        return false;
+        return enabled;
     }
 
     /** {@inheritDoc} */
     @Override public void onDisconnected(IgniteFuture<?> reconnectFut) throws IgniteCheckedException {
         ctx.addNodeAttribute("test", "2");
     }
+
+    /**
+     *
+     */
+    private static class TestSecurityContext implements SecurityContext, Serializable {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public SecuritySubject subject() {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean taskOperationAllowed(String taskClsName, SecurityPermission perm) {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean cacheOperationAllowed(String cacheName, SecurityPermission perm) {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean serviceOperationAllowed(String srvcName, SecurityPermission perm) {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean systemOperationAllowed(SecurityPermission perm) {
+            return true;
+        }
+    }
 }

From b77428d12658b3ab2cdd43ca61ed71d329e83283 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 10 Jan 2017 16:59:17 +0300
Subject: [PATCH 171/516] Do not evict removed entries, otherwise removes can
 be lost.

(cherry picked from commit 55ac6e7)
---
 .../internal/processors/cache/GridCacheMapEntry.java      | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 58b4ae3130c9d..c8b8cd115fefc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -4300,6 +4300,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                     if (obsoleteVersionExtras() != null)
                         return true;
 
+                    // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                    if (cctx.deferredDelete() && deletedUnlocked())
+                        return false;
+
                     CacheObject prev = saveOldValueUnlocked(false);
 
                     if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {
@@ -4358,6 +4362,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                             // Version has changed since entry passed the filter. Do it again.
                             continue;
 
+                        // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                        if (cctx.deferredDelete() && deletedUnlocked())
+                            return false;
+
                         CacheObject prevVal = saveValueForIndexUnlocked();
 
                         if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {

From 29187ef6b663eafe67eaaaf38e4c09fc244ac7aa Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 24 May 2017 17:31:27 +0300
Subject: [PATCH 172/516] Do not evict removed entries, otherwise removes can
 be lost.

Rollback due to test failings.
---
 .../internal/processors/cache/GridCacheMapEntry.java      | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index c8b8cd115fefc..58b4ae3130c9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -4300,10 +4300,6 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                     if (obsoleteVersionExtras() != null)
                         return true;
 
-                    // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
-                    if (cctx.deferredDelete() && deletedUnlocked())
-                        return false;
-
                     CacheObject prev = saveOldValueUnlocked(false);
 
                     if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {
@@ -4362,10 +4358,6 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                             // Version has changed since entry passed the filter. Do it again.
                             continue;
 
-                        // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
-                        if (cctx.deferredDelete() && deletedUnlocked())
-                            return false;
-
                         CacheObject prevVal = saveValueForIndexUnlocked();
 
                         if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {

From 442aac2507210d39b7f30ab8f8d9a3dbe2610cae Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 24 May 2017 18:32:11 +0300
Subject: [PATCH 173/516] IGNITE-5225: Fix NPE caused by changes in
 IGNITE-4577.

(cherry picked from commit d463840)
---
 .../ignite/internal/util/IgniteUtils.java       |  4 ++--
 .../communication/tcp/TcpCommunicationSpi.java  | 17 +++++++++++------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index ba118cb118cea..ca29adf81e1d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1813,11 +1813,11 @@ public static synchronized boolean isLocalHostChanged() throws IOException {
      * @return List of reachable addresses.
      */
     public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
-        final int reachTimeout = 2000;
-
         if (addrs.isEmpty())
             return Collections.emptyList();
 
+        final int reachTimeout = 2000;
+
         if (addrs.size() == 1) {
             InetAddress addr = F.first(addrs);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 81454f827e58f..6dba6b2b5fd0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2336,22 +2336,27 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
 
         Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
 
-        for (InetSocketAddress addr : addrs)
-            allInetAddrs.add(addr.getAddress());
+        for (InetSocketAddress addr : addrs) {
+            // Skip unresolved as addr.getAddress() can return null.
+            if(!addr.isUnresolved())
+                allInetAddrs.add(addr.getAddress());
+        }
 
         List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
 
         if (reachableInetAddrs.size() < allInetAddrs.size()) {
             LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
 
+            List<InetSocketAddress> unreachableInetAddr = new ArrayList<>(allInetAddrs.size() - reachableInetAddrs.size());
+
             for (InetSocketAddress addr : addrs) {
                 if (reachableInetAddrs.contains(addr.getAddress()))
                     addrs0.add(addr);
+                else
+                    unreachableInetAddr.add(addr);
             }
-            for (InetSocketAddress addr : addrs) {
-                if (!reachableInetAddrs.contains(addr.getAddress()))
-                    addrs0.add(addr);
-            }
+
+            addrs0.addAll(unreachableInetAddr);
 
             addrs = addrs0;
         }

From b1736c0bd87d6cfb65f9ef422241e0f1aba04c8d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 24 May 2017 18:48:52 +0300
Subject: [PATCH 174/516] Fixed thread pools incorrect shutdown.

(cherry picked from commit 66cef22)
---
 .../processors/cache/GridCacheAdapter.java    | 12 ++---
 .../ignite/internal/util/IgniteUtils.java     | 45 ++++++++++---------
 2 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 11bf34b221bb1..189f602328894 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1132,14 +1132,14 @@ public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, bool
         if (!F.isEmpty(jobs)) {
             ExecutorService execSvc = null;
 
-            if (jobs.size() > 1) {
-                execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
+            try {
+                if (jobs.size() > 1) {
+                    execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
 
-                for (int i = 1; i < jobs.size(); i++)
-                    execSvc.submit(jobs.get(i));
-            }
+                    for (int i = 1; i < jobs.size(); i++)
+                        execSvc.submit(jobs.get(i));
+                }
 
-            try {
                 jobs.get(0).run();
             }
             finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index ca29adf81e1d3..c2efb953c9135 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -1833,33 +1833,36 @@ public static List<InetAddress> filterReachable(Collection<InetAddress> addrs) {
 
         ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, addrs.size()));
 
-        for (final InetAddress addr : addrs) {
-            futs.add(executor.submit(new Runnable() {
-                @Override public void run() {
-                    if (reachable(addr, reachTimeout)) {
-                        synchronized (res) {
-                            res.add(addr);
+        try {
+            for (final InetAddress addr : addrs) {
+                futs.add(executor.submit(new Runnable() {
+                    @Override public void run() {
+                        if (reachable(addr, reachTimeout)) {
+                            synchronized (res) {
+                                res.add(addr);
+                            }
                         }
                     }
-                }
-            }));
-        }
-
-        for (Future<?> fut : futs) {
-            try {
-                fut.get();
+                }));
             }
-            catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
 
-                throw new IgniteException("Thread has been interrupted.", e);
-            }
-            catch (ExecutionException e) {
-                throw new IgniteException(e);
+            for (Future<?> fut : futs) {
+                try {
+                    fut.get();
+                }
+                catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+
+                    throw new IgniteException("Thread has been interrupted.", e);
+                }
+                catch (ExecutionException e) {
+                    throw new IgniteException(e);
+                }
             }
         }
-
-        executor.shutdown();
+        finally {
+            executor.shutdown();
+        }
 
         return res;
     }

From 15d94b432fdfe458a826df6ad3c30a0408a93f49 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 25 May 2017 14:27:08 +0300
Subject: [PATCH 175/516] Backport of IGNITE-4336: Manual rebalance can't be
 requested twice. (cherry picked from commit 9a691c4)

---
 .../processors/cache/GridCacheAdapter.java    |  5 +-
 .../GridCachePartitionExchangeManager.java    | 11 ++-
 .../processors/cache/GridCachePreloader.java  |  8 +-
 .../cache/GridCachePreloaderAdapter.java      |  7 +-
 .../processors/cache/IgniteCacheProxy.java    |  4 +-
 .../preloader/GridDhtPartitionDemander.java   | 46 ++++++++++--
 .../GridDhtPartitionsExchangeFuture.java      | 14 +++-
 .../dht/preloader/GridDhtPreloader.java       |  8 +-
 .../cache/CacheRebalancingSelfTest.java       | 75 +++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |  3 +
 10 files changed, 156 insertions(+), 25 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index b49becaacbc74..f10406fce2a80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -86,6 +86,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter;
@@ -4535,9 +4536,7 @@ protected Object readResolve() throws ObjectStreamException {
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<?> rebalance() {
-        ctx.preloader().forcePreload();
-
-        return ctx.preloader().syncFuture();
+        return ctx.preloader().forceRebalance();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 679395727123a..c175c43ae2f2a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -723,9 +723,13 @@ public void forceDummyExchange(boolean reassign,
      *
      * @param exchFut Exchange future.
      */
-    public void forcePreloadExchange(GridDhtPartitionsExchangeFuture exchFut) {
+    public IgniteInternalFuture<Boolean> forceRebalance(GridDhtPartitionsExchangeFuture exchFut) {
+        GridFutureAdapter<Boolean> fut = new GridFutureAdapter<>();
+
         exchWorker.addFuture(
-            new GridDhtPartitionsExchangeFuture(cctx, exchFut.discoveryEvent(), exchFut.exchangeId()));
+            new GridDhtPartitionsExchangeFuture(cctx, exchFut.discoveryEvent(), exchFut.exchangeId(), fut));
+
+        return fut;
     }
 
     /**
@@ -1815,7 +1819,8 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                 Runnable cur = cacheCtx.preloader().addAssignments(assigns,
                                     forcePreload,
                                     cnt,
-                                    r);
+                                    r,
+                                    exchFut.forcedRebalanceFuture());
 
                                 if (cur != null) {
                                     rebList.add(U.maskName(cacheCtx.name()));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index 3c4456d5984d9..0c2869101aa79 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
@@ -90,7 +91,8 @@ public interface GridCachePreloader {
     public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
         boolean forcePreload,
         int cnt,
-        Runnable next);
+        Runnable next,
+        @Nullable GridFutureAdapter<Boolean> forcedRebFut);
 
     /**
      * @param p Preload predicate.
@@ -150,9 +152,9 @@ public IgniteInternalFuture<Object> request(GridNearAtomicAbstractUpdateRequest
         AffinityTopologyVersion topVer);
 
     /**
-     * Force preload process.
+     * Force Rebalance process.
      */
-    public void forcePreload();
+    public IgniteInternalFuture<Boolean> forceRebalance();
 
     /**
      * Unwinds undeploys.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index 656a960b56b9f..8ae67215dabc9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
@@ -88,8 +89,8 @@ public GridCachePreloaderAdapter(GridCacheContext<?, ?> cctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public void forcePreload() {
-        // No-op.
+    @Override public IgniteInternalFuture<Boolean> forceRebalance() {
+        return new GridFinishedFuture<>(true);
     }
 
     /** {@inheritDoc} */
@@ -166,7 +167,7 @@ public GridCachePreloaderAdapter(GridCacheContext<?, ?> cctx) {
 
     /** {@inheritDoc} */
     @Override public Runnable addAssignments(GridDhtPreloaderAssignments assignments, boolean forcePreload,
-        int cnt, Runnable next) {
+        int cnt, Runnable next, @Nullable GridFutureAdapter<Boolean> forcedRebFut) {
         return null;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
index 873c8221ae921..13816709557a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
@@ -2329,9 +2329,7 @@ private void onLeave(GridCacheGateway<K, V> gate) {
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<?> rebalance() {
-        ctx.preloader().forcePreload();
-
-        return new IgniteFutureImpl<>(ctx.preloader().syncFuture());
+        return new IgniteFutureImpl<>(ctx.preloader().forceRebalance());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index a6808c73577e1..9ece00cd01049 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
 import org.apache.ignite.internal.util.GridLeanSet;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -64,6 +65,7 @@
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.spi.IgniteSpiException;
@@ -216,9 +218,9 @@ void preloadPredicate(IgnitePredicate<GridCacheEntryInfo> preloadPred) {
     }
 
     /**
-     * Force preload.
+     * Force Rebalance.
      */
-    void forcePreload() {
+    IgniteInternalFuture<Boolean> forceRebalance() {
         GridTimeoutObject obj = lastTimeoutObj.getAndSet(null);
 
         if (obj != null)
@@ -230,14 +232,31 @@ void forcePreload() {
             if (log.isDebugEnabled())
                 log.debug("Forcing rebalance event for future: " + exchFut);
 
+            final GridFutureAdapter<Boolean> fut = new GridFutureAdapter<>();
+
             exchFut.listen(new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
                 @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> t) {
-                    cctx.shared().exchange().forcePreloadExchange(exchFut);
+                    IgniteInternalFuture<Boolean> fut0 = cctx.shared().exchange().forceRebalance(exchFut);
+
+                    fut0.listen(new IgniteInClosure<IgniteInternalFuture<Boolean>>() {
+                        @Override public void apply(IgniteInternalFuture<Boolean> future) {
+                            try {
+                                fut.onDone(future.get());
+                            }
+                            catch (Exception e) {
+                                fut.onDone(e);
+                            }
+                        }
+                    });
                 }
             });
+
+            return fut;
         }
         else if (log.isDebugEnabled())
             log.debug("Ignoring force rebalance request (no topology event happened yet).");
+
+        return new GridFinishedFuture<>(true);
     }
 
     /**
@@ -274,16 +293,20 @@ void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
      * @param assigns Assignments.
      * @param force {@code True} if dummy reassign.
      * @param cnt Counter.
+     * * @param forcedRebFut External future for forced rebalance.
      * @param next Runnable responsible for cache rebalancing start.
      * @return Rebalancing runnable.
      */
     Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
         boolean force,
         int cnt,
-        final Runnable next) {
+        final Runnable next,
+        @Nullable final GridFutureAdapter<Boolean> forcedRebFut) {
         if (log.isDebugEnabled())
             log.debug("Adding partition assignments: " + assigns);
 
+        assert force == (forcedRebFut != null);
+
         long delay = cctx.config().getRebalanceDelay();
 
         if (delay == 0 || force) {
@@ -301,6 +324,19 @@ Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
                 });
             }
 
+            if (forcedRebFut != null) {
+                fut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
+                    @Override public void apply(IgniteInternalFuture<Boolean> future) {
+                        try {
+                            forcedRebFut.onDone(future.get());
+                        }
+                        catch (Exception e) {
+                            forcedRebFut.onDone(e);
+                        }
+                    }
+                });
+            }
+
             rebalanceFut = fut;
 
             fut.sendRebalanceStartedEvent();
@@ -383,7 +419,7 @@ else if (delay > 0) {
                 @Override public void onTimeout() {
                     exchFut.listen(new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
                         @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> f) {
-                            cctx.shared().exchange().forcePreloadExchange(exchFut);
+                            cctx.shared().exchange().forceRebalance(exchFut);
                         }
                     });
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 9aa0755269b9f..2245d17e83523 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -197,6 +197,9 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** */
     private boolean centralizedAff;
 
+    /** Forced Rebalance future. */
+    private GridFutureAdapter<Boolean> forcedRebFut;
+
     /**
      * Dummy future created to trigger reassignments if partition
      * topology changed while preloading.
@@ -230,15 +233,17 @@ public GridDhtPartitionsExchangeFuture(
      * @param cctx Cache context.
      * @param discoEvt Discovery event.
      * @param exchId Exchange id.
+     * @param forcedRebFut Forced Rebalance future.
      */
     public GridDhtPartitionsExchangeFuture(GridCacheSharedContext cctx, DiscoveryEvent discoEvt,
-        GridDhtPartitionExchangeId exchId) {
+        GridDhtPartitionExchangeId exchId, GridFutureAdapter<Boolean> forcedRebFut) {
         dummy = false;
         forcePreload = true;
 
         this.exchId = exchId;
         this.discoEvt = discoEvt;
         this.cctx = cctx;
+        this.forcedRebFut = forcedRebFut;
 
         reassign = true;
 
@@ -409,6 +414,13 @@ public GridDhtPartitionExchangeId exchangeId() {
         return exchId;
     }
 
+    /**
+     * @return Forced Rebalance future.
+     */
+    @Nullable public GridFutureAdapter<Boolean> forcedRebalanceFuture() {
+        return forcedRebFut;
+    }
+
     /**
      * @return {@code true} if entered to busy state.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 692e7c0de45a6..4aff4d5a71f42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -413,8 +413,8 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
 
     /** {@inheritDoc} */
     @Override public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
-        boolean forcePreload, int cnt, Runnable next) {
-        return demander.addAssignments(assignments, forcePreload, cnt, next);
+        boolean forcePreload, int cnt, Runnable next,  @Nullable GridFutureAdapter<Boolean> forcedRebFut) {
+        return demander.addAssignments(assignments, forcePreload, cnt, next, forcedRebFut);
     }
 
     /**
@@ -728,8 +728,8 @@ private GridDhtFuture<Object> request0(Collection<KeyCacheObject> keys, Affinity
     }
 
     /** {@inheritDoc} */
-    @Override public void forcePreload() {
-        demander.forcePreload();
+    @Override public IgniteInternalFuture<Boolean> forceRebalance() {
+        return demander.forceRebalance();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
new file mode 100644
index 0000000000000..8d1f67af2c33f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for rebalancing.
+ */
+public class CacheRebalancingSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCacheConfiguration(new CacheConfiguration());
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRebalanceFuture() throws Exception {
+        IgniteEx ignite0 = startGrid(0);
+        startGrid(1);
+
+        IgniteCache<Object, Object> cache = ignite0.cache(null);
+
+        IgniteFuture fut1 = cache.rebalance();
+
+        fut1.get();
+
+        startGrid(2);
+
+        IgniteFuture fut2 = cache.rebalance();
+
+        assert internalFuture(fut2) != internalFuture(fut1);
+
+        fut2.get();
+    }
+
+    /**
+     * @param future Future.
+     * @return Internal future.
+     */
+    private static IgniteInternalFuture internalFuture(IgniteFuture future) {
+        assert future instanceof IgniteFutureImpl;
+
+        return ((IgniteFutureImpl)future).internalFuture();
+    }
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 421676654e7d5..58a26e76ddd2f 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.CacheKeepBinaryTransactionTest;
 import org.apache.ignite.internal.processors.cache.CacheNearReaderUpdateTest;
+import org.apache.ignite.internal.processors.cache.CacheRebalancingSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheSerializableTransactionsTest;
 import org.apache.ignite.internal.processors.cache.GridCacheSwapSpaceSpiConsistencySelfTest;
 import org.apache.ignite.internal.processors.cache.EntryVersionConsistencyReadThroughTest;
@@ -64,6 +65,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(GridCacheOffHeapCleanupTest.class);
 
+        suite.addTestSuite(CacheRebalancingSelfTest.class);
+
         return suite;
     }
 }

From 3a12fee29625de8d75a291e39b7d52c5f5111fb4 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 25 May 2017 19:38:59 +0300
Subject: [PATCH 176/516] Minors fix segmented indices snapshots.

---
 .../processors/query/h2/opt/GridH2Table.java       | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
index 6b087c2cdf91e..27bcf93bbcbb2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
@@ -311,7 +311,7 @@ public void snapshotIndexes(GridH2QueryContext qctx, int segment) {
             segmentSnapshot = actualSnapshot.get(segment);
 
             if (segmentSnapshot != null) { // Reuse existing snapshot without locking.
-                segmentSnapshot = doSnapshotIndexes(segmentSnapshot, qctx);
+                segmentSnapshot = doSnapshotIndexes(segment, segmentSnapshot, qctx);
 
                 if (segmentSnapshot != null)
                     return; // Reused successfully.
@@ -328,10 +328,10 @@ public void snapshotIndexes(GridH2QueryContext qctx, int segment) {
             segmentSnapshot = actualSnapshot.get(segment);
 
             if (segmentSnapshot != null) // Try reusing.
-                segmentSnapshot = doSnapshotIndexes(segmentSnapshot, qctx);
+                segmentSnapshot = doSnapshotIndexes(segment, segmentSnapshot, qctx);
 
             if (segmentSnapshot == null) { // Reuse failed, produce new snapshots.
-                segmentSnapshot = doSnapshotIndexes(null, qctx);
+                segmentSnapshot = doSnapshotIndexes(segment, null, qctx);
 
                 assert segmentSnapshot != null;
 
@@ -390,12 +390,13 @@ public Lock lock(boolean exclusive, long waitMillis) {
      * Must be called inside of write lock because when using multiple indexes we have to ensure that all of them have
      * the same contents at snapshot taking time.
      *
+     * @param segment id of index segment snapshot.
      * @param segmentSnapshot snapshot to be reused.
      * @param qctx Query context.
      * @return New indexes data snapshot.
      */
     @SuppressWarnings("unchecked")
-    private Object[] doSnapshotIndexes(Object[] segmentSnapshot, GridH2QueryContext qctx) {
+    private Object[] doSnapshotIndexes(int segment, Object[] segmentSnapshot, GridH2QueryContext qctx) {
         assert snapshotEnabled;
 
         if (segmentSnapshot == null) // Nothing to reuse, create new snapshots.
@@ -418,7 +419,7 @@ private Object[] doSnapshotIndexes(Object[] segmentSnapshot, GridH2QueryContext
                     index(j).releaseSnapshot();
 
                 // Drop invalidated snapshot.
-                actualSnapshot.compareAndSet(threadLocalSegmentId(), segmentSnapshot, null);
+                actualSnapshot.compareAndSet(segment, segmentSnapshot, null);
 
                 return null;
             }
@@ -611,7 +612,8 @@ else if (old != null) // Row was not replaced, need to remove manually.
             }
 
             // The snapshot is not actual after update.
-            actualSnapshot.set(pk.segmentForRow(row), null);
+            if (snapshotEnabled)
+                actualSnapshot.set(pk.segmentForRow(row), null);
 
             return true;
         }

From 7362d3c692c28245b193658d727b20caa62ffd38 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 25 May 2017 20:13:01 +0300
Subject: [PATCH 177/516] Merge compilation fix

---
 ...niteClientCacheInitializationFailTest.java | 29 +++++++++++++++----
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
index 63dd57527183b..948461e19c482 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
@@ -242,16 +243,21 @@ private static class FailedIndexing implements GridQueryIndexing {
         }
 
         /** {@inheritDoc} */
-        @Override public GridQueryFieldsResult queryLocalSqlFields(@Nullable String spaceName, String qry,
-            Collection<Object> params, IndexingQueryFilter filter, boolean enforceJoinOrder, int timeout,
-            GridQueryCancel cancel) throws IgniteCheckedException {
+        @Override public <K, V> QueryCursor<List<?>> queryLocalSqlFields(GridCacheContext<?, ?> cctx,
+            SqlFieldsQuery qry,
+            IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException {
             return null;
         }
 
         /** {@inheritDoc} */
-        @Override public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable String spaceName,
-            String qry, String alias, Collection<Object> params, GridQueryTypeDescriptor type,
-            IndexingQueryFilter filter) throws IgniteCheckedException {
+        @Override public long streamUpdateQuery(@Nullable String spaceName, String qry,
+            @Nullable Object[] params, IgniteDataStreamer<?, ?> streamer) throws IgniteCheckedException {
+            return 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public <K, V> QueryCursor<Cache.Entry<K, V>> queryLocalSql(GridCacheContext<?, ?> cctx, SqlQuery qry,
+            IndexingQueryFilter filter, boolean keepBinary) throws IgniteCheckedException {
             return null;
         }
 
@@ -328,6 +334,11 @@ private static class FailedIndexing implements GridQueryIndexing {
             return null;
         }
 
+        /** {@inheritDoc} */
+        @Override public String space(String schemaName) {
+            return null;
+        }
+
         /** {@inheritDoc} */
         @Override public Collection<GridRunningQueryInfo> runningQueries(long duration) {
             return null;
@@ -342,5 +353,11 @@ private static class FailedIndexing implements GridQueryIndexing {
         @Override public void cancelAllQueries() {
 
         }
+
+        /** {@inheritDoc} */
+        @Override public IgniteDataStreamer<?, ?> createStreamer(String spaceName, PreparedStatement nativeStmt,
+            long autoFlushFreq, int nodeBufSize, int nodeParOps, boolean allowOverwrite) {
+            return null;
+        }
     }
 }

From 26072dffb8f5b28693731f8367872a8e1e6dfe7e Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Thu, 18 May 2017 19:40:09 +0300
Subject: [PATCH 178/516] ignite-5203 Simple BLOB support added

---
 .../JdbcAbstractDmlStatementSelfTest.java     |  54 +-
 .../JdbcAbstractUpdateStatementSelfTest.java  |  11 +-
 .../ignite/internal/jdbc2/JdbcBlobTest.java   | 485 ++++++++++++++++++
 .../jdbc2/JdbcInsertStatementSelfTest.java    |  12 +-
 .../jdbc2/JdbcMergeStatementSelfTest.java     |  12 +-
 .../jdbc2/JdbcPreparedStatementSelfTest.java  |  46 ++
 .../jdbc/suite/IgniteJdbcDriverTestSuite.java |   6 +-
 .../ignite/internal/jdbc2/JdbcBlob.java       | 188 +++++++
 .../ignite/internal/jdbc2/JdbcConnection.java |   2 +-
 .../internal/jdbc2/JdbcPreparedStatement.java |   4 +-
 .../ignite/internal/jdbc2/JdbcResultSet.java  |   8 +-
 11 files changed, 799 insertions(+), 29 deletions(-)
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcBlobTest.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBlob.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
index 4a97aefcfcbe0..76597f1668bc8 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
@@ -18,9 +18,12 @@
 package org.apache.ignite.internal.jdbc2;
 
 import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.sql.Blob;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Collections;
 import org.apache.ignite.cache.CachePeekMode;
@@ -44,6 +47,9 @@
  * Statement test.
  */
 public abstract class JdbcAbstractDmlStatementSelfTest extends GridCommonAbstractTest {
+    /** UTF 16 character set name. */
+    private static final String UTF_16 = "UTF-16"; // RAWTOHEX function use UTF-16 for conversion strings to byte arrays.
+
     /** IP finder. */
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
@@ -54,7 +60,7 @@ public abstract class JdbcAbstractDmlStatementSelfTest extends GridCommonAbstrac
     static final String BASE_URL_BIN = CFG_URL_PREFIX + "modules/clients/src/test/config/jdbc-bin-config.xml";
 
     /** SQL SELECT query for verification. */
-    private static final String SQL_SELECT = "select _key, id, firstName, lastName, age from Person";
+    private static final String SQL_SELECT = "select _key, id, firstName, lastName, age, data from Person";
 
     /** Connection. */
     protected Connection conn;
@@ -117,6 +123,7 @@ IgniteConfiguration getBinaryConfiguration(String gridName) throws Exception {
         e.addQueryField("age", Integer.class.getName(), null);
         e.addQueryField("firstName", String.class.getName(), null);
         e.addQueryField("lastName", String.class.getName(), null);
+        e.addQueryField("data", byte[].class.getName(), null);
 
         ccfg.setQueryEntities(Collections.singletonList(e));
 
@@ -165,6 +172,7 @@ protected String getCfgUrl() {
                         assertEquals("John", rs.getString("firstName"));
                         assertEquals("White", rs.getString("lastName"));
                         assertEquals(25, rs.getInt("age"));
+                        assertEquals("White", str(getBytes(rs.getBlob("data"))));
                         break;
 
                     case 2:
@@ -172,6 +180,7 @@ protected String getCfgUrl() {
                         assertEquals("Joe", rs.getString("firstName"));
                         assertEquals("Black", rs.getString("lastName"));
                         assertEquals(35, rs.getInt("age"));
+                        assertEquals("Black", str(getBytes(rs.getBlob("data"))));
                         break;
 
                     case 3:
@@ -179,6 +188,7 @@ protected String getCfgUrl() {
                         assertEquals("Mike", rs.getString("firstName"));
                         assertEquals("Green", rs.getString("lastName"));
                         assertEquals(40, rs.getInt("age"));
+                        assertEquals("Green", str(getBytes(rs.getBlob("data"))));
                         break;
 
                     case 4:
@@ -186,6 +196,7 @@ protected String getCfgUrl() {
                         assertEquals("Leah", rs.getString("firstName"));
                         assertEquals("Grey", rs.getString("lastName"));
                         assertEquals(22, rs.getInt("age"));
+                        assertEquals("Grey", str(getBytes(rs.getBlob("data"))));
                         break;
 
                     default:
@@ -199,6 +210,42 @@ protected String getCfgUrl() {
         assertEquals(0, grid(0).cache(null).size(CachePeekMode.ALL));
     }
 
+    /**
+     * @param str String.
+     */
+    static byte[] getBytes(String str) {
+        try {
+            return str.getBytes(UTF_16);
+        }
+        catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * @param blob Blob.
+     */
+    static byte[] getBytes(Blob blob) {
+        try {
+            return blob.getBytes(1, (int)blob.length());
+        }
+        catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * @param arr Array.
+     */
+    static String str(byte[] arr) {
+        try {
+            return new String(arr, UTF_16);
+        }
+        catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      * Person.
      */
@@ -220,6 +267,10 @@ static class Person implements Serializable {
         @QuerySqlField
         private final int age;
 
+        /** Binary data. */
+        @QuerySqlField
+        private final byte[] data;
+
         /**
          * @param id ID.
          * @param firstName First name.
@@ -235,6 +286,7 @@ static class Person implements Serializable {
             this.firstName = firstName;
             this.lastName = lastName;
             this.age = age;
+            this.data = getBytes(lastName);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractUpdateStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractUpdateStatementSelfTest.java
index a20b8157ef1e8..ace1be665614c 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractUpdateStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractUpdateStatementSelfTest.java
@@ -19,12 +19,15 @@
 
 import java.sql.Statement;
 
+/**
+ *
+ */
 public abstract class JdbcAbstractUpdateStatementSelfTest extends JdbcAbstractDmlStatementSelfTest {
     /** SQL query to populate cache. */
-    private static final String ITEMS_SQL = "insert into Person(_key, id, firstName, lastName, age) values " +
-        "('p1', 1, 'John', 'White', 25), " +
-        "('p2', 2, 'Joe', 'Black', 35), " +
-        "('p3', 3, 'Mike', 'Green', 40)";
+    private static final String ITEMS_SQL = "insert into Person(_key, id, firstName, lastName, age, data) values " +
+        "('p1', 1, 'John', 'White', 25, RAWTOHEX('White')), " +
+        "('p2', 2, 'Joe', 'Black', 35, RAWTOHEX('Black')), " +
+        "('p3', 3, 'Mike', 'Green', 40, RAWTOHEX('Green'))";
 
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcBlobTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcBlobTest.java
new file mode 100644
index 0000000000000..9e0e0d2f6aab1
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcBlobTest.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.SQLException;
+import java.util.Arrays;
+import junit.framework.TestCase;
+
+/**
+ *
+ */
+public class JdbcBlobTest extends TestCase {
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLength() throws Exception {
+        JdbcBlob blob = new JdbcBlob(new byte[16]);
+
+        assertEquals(16, (int)blob.length());
+
+        blob.free();
+
+        try {
+            blob.length();
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testGetBytes() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        try {
+            blob.getBytes(0, 16);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.getBytes(17, 16);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.getBytes(1, -1);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        byte[] res = blob.getBytes(1, 0);
+        assertEquals(0, res.length);
+
+        assertTrue(Arrays.equals(arr, blob.getBytes(1, 16)));
+
+        res = blob.getBytes(1, 20);
+        assertEquals(16, res.length);
+        assertTrue(Arrays.equals(arr, res));
+
+        res = blob.getBytes(1, 10);
+        assertEquals(10, res.length);
+        assertEquals(0, res[0]);
+        assertEquals(9, res[9]);
+
+        res = blob.getBytes(7, 10);
+        assertEquals(10, res.length);
+        assertEquals(6, res[0]);
+        assertEquals(15, res[9]);
+
+        res = blob.getBytes(7, 20);
+        assertEquals(10, res.length);
+        assertEquals(6, res[0]);
+        assertEquals(15, res[9]);
+
+        res = blob.getBytes(1, 0);
+        assertEquals(0, res.length);
+
+        blob.free();
+
+        try {
+            blob.getBytes(1, 16);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testGetBinaryStream() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        InputStream is = blob.getBinaryStream();
+
+        byte[] res = readBytes(is);
+
+        assertTrue(Arrays.equals(arr, res));
+
+        blob.free();
+
+        try {
+            blob.getBinaryStream();
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testGetBinaryStreamWithParams() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        try {
+            blob.getBinaryStream(0, arr.length);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.getBinaryStream(1, 0);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.getBinaryStream(17, arr.length);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.getBinaryStream(1, arr.length + 1);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        InputStream is = blob.getBinaryStream(1, arr.length);
+        byte[] res = readBytes(is);
+        assertTrue(Arrays.equals(arr, res));
+
+        is = blob.getBinaryStream(1, 10);
+        res = readBytes(is);
+        assertEquals(10, res.length);
+        assertEquals(0, res[0]);
+        assertEquals(9, res[9]);
+
+        is = blob.getBinaryStream(6, 10);
+        res = readBytes(is);
+        assertEquals(10, res.length);
+        assertEquals(5, res[0]);
+        assertEquals(14, res[9]);
+
+        blob.free();
+
+        try {
+            blob.getBinaryStream(1, arr.length);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPositionBytePattern() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        assertEquals(-1, blob.position(new byte[] {1, 2, 3}, 0));
+        assertEquals(-1, blob.position(new byte[] {1, 2, 3}, arr.length + 1));
+        assertEquals(-1, blob.position(new byte[0], 1));
+        assertEquals(-1, blob.position(new byte[17], 1));
+        assertEquals(-1, blob.position(new byte[] {3, 2, 1}, 1));
+        assertEquals(1, blob.position(new byte[] {0, 1, 2}, 1));
+        assertEquals(2, blob.position(new byte[] {1, 2, 3}, 1));
+        assertEquals(2, blob.position(new byte[] {1, 2, 3}, 2));
+        assertEquals(-1, blob.position(new byte[] {1, 2, 3}, 3));
+        assertEquals(14, blob.position(new byte[] {13, 14, 15}, 3));
+        assertEquals(-1, blob.position(new byte[] {0, 1, 3}, 1));
+        assertEquals(-1, blob.position(new byte[] {0, 2, 3}, 1));
+        assertEquals(-1, blob.position(new byte[] {1, 2, 4}, 1));
+
+        blob.free();
+
+        try {
+            blob.position(new byte[] {0, 1, 2}, 1);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPositionBlobPattern() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {1, 2, 3}), 0));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {1, 2, 3}), arr.length + 1));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[0]), 1));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[17]), 1));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {3, 2, 1}), 1));
+        assertEquals(1, blob.position(new JdbcBlob(new byte[] {0, 1, 2}), 1));
+        assertEquals(2, blob.position(new JdbcBlob(new byte[] {1, 2, 3}), 1));
+        assertEquals(2, blob.position(new JdbcBlob(new byte[] {1, 2, 3}), 2));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {1, 2, 3}), 3));
+        assertEquals(14, blob.position(new JdbcBlob(new byte[] {13, 14, 15}), 3));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {0, 1, 3}), 1));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {0, 2, 3}), 1));
+        assertEquals(-1, blob.position(new JdbcBlob(new byte[] {1, 2, 4}), 1));
+
+        blob.free();
+
+        try {
+            blob.position(new JdbcBlob(new byte[] {0, 1, 2}), 1);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSetBytes() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        try {
+            blob.setBytes(0, new byte[4]);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.setBytes(17, new byte[4]);
+
+            fail();
+        }
+        catch (ArrayIndexOutOfBoundsException e) {
+            // No-op.
+        }
+
+        assertEquals(4, blob.setBytes(1, new byte[] {3, 2, 1, 0}));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 4, 5, 6, 7}, blob.getBytes(1, arr.length)));
+
+        assertEquals(4, blob.setBytes(5, new byte[] {7, 6, 5, 4}));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 7, 6, 5, 4}, blob.getBytes(1, arr.length)));
+
+        assertEquals(4, blob.setBytes(7, new byte[] {8, 9, 10, 11}));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 7, 6, 8, 9, 10, 11}, blob.getBytes(1, (int)blob.length())));
+
+        blob = new JdbcBlob(new byte[] {15, 16});
+        assertEquals(8, blob.setBytes(1, new byte[] {0, 1, 2, 3, 4, 5, 6, 7}));
+        assertTrue(Arrays.equals(new byte[] {0, 1, 2, 3, 4, 5, 6, 7}, blob.getBytes(1, (int)blob.length())));
+
+        blob.free();
+
+        try {
+            blob.setBytes(1, new byte[] {0, 1, 2});
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSetBytesWithOffsetAndLength() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        try {
+            blob.setBytes(0, new byte[4], 0, 2);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.setBytes(17, new byte[4], 0, 2);
+
+            fail();
+        }
+        catch (ArrayIndexOutOfBoundsException e) {
+            // No-op.
+        }
+
+        try {
+            blob.setBytes(1, new byte[4], -1, 2);
+
+            fail();
+        }
+        catch (ArrayIndexOutOfBoundsException e) {
+            // No-op.
+        }
+
+        try {
+            blob.setBytes(1, new byte[4], 0, 5);
+
+            fail();
+        }
+        catch (ArrayIndexOutOfBoundsException e) {
+            // No-op.
+        }
+
+        assertEquals(4, blob.setBytes(1, new byte[] {3, 2, 1, 0}, 0, 4));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 4, 5, 6, 7}, blob.getBytes(1, arr.length)));
+
+        assertEquals(4, blob.setBytes(5, new byte[] {7, 6, 5, 4}, 0, 4));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 7, 6, 5, 4}, blob.getBytes(1, arr.length)));
+
+        assertEquals(4, blob.setBytes(7, new byte[] {8, 9, 10, 11}, 0, 4));
+        assertTrue(Arrays.equals(new byte[] {3, 2, 1, 0, 7, 6, 8, 9, 10, 11}, blob.getBytes(1, (int)blob.length())));
+
+        assertEquals(2, blob.setBytes(1, new byte[] {3, 2, 1, 0}, 2, 2));
+        assertTrue(Arrays.equals(new byte[] {1, 0, 1, 0, 7, 6, 8, 9, 10, 11}, blob.getBytes(1, (int)blob.length())));
+
+        assertEquals(2, blob.setBytes(9, new byte[] {3, 2, 1, 0}, 1, 2));
+        assertTrue(Arrays.equals(new byte[] {1, 0, 1, 0, 7, 6, 8, 9, 2, 1}, blob.getBytes(1, (int)blob.length())));
+
+        assertEquals(3, blob.setBytes(9, new byte[] {3, 2, 1, 0}, 0, 3));
+        assertTrue(Arrays.equals(new byte[] {1, 0, 1, 0, 7, 6, 8, 9, 3, 2, 1}, blob.getBytes(1, (int)blob.length())));
+
+        blob = new JdbcBlob(new byte[] {15, 16});
+        assertEquals(8, blob.setBytes(1, new byte[] {0, 1, 2, 3, 4, 5, 6, 7}, 0, 8));
+        assertTrue(Arrays.equals(new byte[] {0, 1, 2, 3, 4, 5, 6, 7}, blob.getBytes(1, (int)blob.length())));
+
+        blob.free();
+
+        try {
+            blob.setBytes(1, new byte[] {0, 1, 2}, 0, 2);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTruncate() throws Exception {
+        byte[] arr = new byte[] {0, 1, 2, 3, 4, 5, 6, 7};
+
+        JdbcBlob blob = new JdbcBlob(arr);
+
+        try {
+            blob.truncate(-1);
+
+            fail();
+        }
+        catch(SQLException e) {
+            // No-op.
+        }
+
+        try {
+            blob.truncate(arr.length + 1);
+
+            fail();
+        }
+        catch(SQLException e) {
+            // No-op.
+        }
+
+        blob.truncate(4);
+        assertTrue(Arrays.equals(new byte[] {0, 1, 2, 3}, blob.getBytes(1, (int)blob.length())));
+
+        blob.truncate(0);
+        assertEquals(0, (int)blob.length());
+
+        blob.free();
+
+        try {
+            blob.truncate(0);
+
+            fail();
+        }
+        catch (SQLException e) {
+            // No-op.
+            System.out.println();
+        }
+    }
+
+    /**
+     * @param is Input stream.
+     */
+    private static byte[] readBytes(InputStream is) throws IOException {
+        byte[] tmp = new byte[16];
+
+        int i = 0;
+        int read;
+        int cnt = 0;
+
+        while ((read = is.read()) != -1) {
+            tmp[i++] = (byte)read;
+            cnt++;
+        }
+
+        byte[] res = new byte[cnt];
+
+        System.arraycopy(tmp, 0, res, 0, cnt);
+
+        return res;
+    }
+}
\ No newline at end of file
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java
index 7fc92de20595c..9a44006dd6055 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java
@@ -31,14 +31,14 @@
  */
 public class JdbcInsertStatementSelfTest extends JdbcAbstractDmlStatementSelfTest {
     /** SQL query. */
-    private static final String SQL = "insert into Person(_key, id, firstName, lastName, age) values " +
-        "('p1', 1, 'John', 'White', 25), " +
-        "('p2', 2, 'Joe', 'Black', 35), " +
-        "('p3', 3, 'Mike', 'Green', 40)";
+    private static final String SQL = "insert into Person(_key, id, firstName, lastName, age, data) values " +
+        "('p1', 1, 'John', 'White', 25, RAWTOHEX('White')), " +
+        "('p2', 2, 'Joe', 'Black', 35, RAWTOHEX('Black')), " +
+        "('p3', 3, 'Mike', 'Green', 40, RAWTOHEX('Green'))";
 
     /** SQL query. */
-    private static final String SQL_PREPARED = "insert into Person(_key, id, firstName, lastName, age) values " +
-        "(?, ?, ?, ?, ?), (?, ?, ?, ?, ?)";
+    private static final String SQL_PREPARED = "insert into Person(_key, id, firstName, lastName, age, data) values " +
+        "(?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?)";
 
     /** Statement. */
     private Statement stmt;
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java
index ecf6032db745f..1c93239ddf518 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java
@@ -26,14 +26,14 @@
  */
 public class JdbcMergeStatementSelfTest extends JdbcAbstractDmlStatementSelfTest {
     /** SQL query. */
-    private static final String SQL = "merge into Person(_key, id, firstName, lastName, age) values " +
-        "('p1', 1, 'John', 'White', 25), " +
-        "('p2', 2, 'Joe', 'Black', 35), " +
-        "('p3', 3, 'Mike', 'Green', 40)";
+    private static final String SQL = "merge into Person(_key, id, firstName, lastName, age, data) values " +
+        "('p1', 1, 'John', 'White', 25, RAWTOHEX('White')), " +
+        "('p2', 2, 'Joe', 'Black', 35, RAWTOHEX('Black')), " +
+        "('p3', 3, 'Mike', 'Green', 40, RAWTOHEX('Green'))";
 
     /** SQL query. */
-    protected static final String SQL_PREPARED = "merge into Person(_key, id, firstName, lastName, age) values " +
-        "(?, ?, ?, ?, ?), (?, ?, ?, ?, ?)";
+    protected static final String SQL_PREPARED = "merge into Person(_key, id, firstName, lastName, age, data) values " +
+        "(?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?)";
 
     /** Statement. */
     protected Statement stmt;
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
index ea586b297a579..edc6031e7d3ec 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
@@ -98,6 +98,7 @@ public class JdbcPreparedStatementSelfTest extends GridCommonAbstractTest {
         o.bigVal = new BigDecimal(1);
         o.strVal = "str";
         o.arrVal = new byte[] {1};
+        o.blobVal = new byte[] {1};
         o.dateVal = new Date(1);
         o.timeVal = new Time(1);
         o.tsVal = new Timestamp(1);
@@ -507,6 +508,47 @@ public void testArray() throws Exception {
         assert cnt == 1;
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBlob() throws Exception {
+        stmt = conn.prepareStatement("select * from TestObject where blobVal is not distinct from ?");
+
+        Blob blob = conn.createBlob();
+
+        blob.setBytes(1, new byte[] {1});
+
+        stmt.setBlob(1, blob);
+
+        ResultSet rs = stmt.executeQuery();
+
+        int cnt = 0;
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assert rs.getInt("id") == 1;
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+
+        stmt.setNull(1, BINARY);
+
+        rs = stmt.executeQuery();
+
+        cnt = 0;
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assert rs.getInt("id") == 2;
+
+            cnt++;
+        }
+
+        assert cnt == 1;
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -704,6 +746,10 @@ private static class TestObject implements Serializable {
         @QuerySqlField
         private byte[] arrVal;
 
+        /** */
+        @QuerySqlField
+        private byte[] blobVal;
+
         /** */
         @QuerySqlField
         private Date dateVal;
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index a28d29e00c52b..2718013f9d014 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.jdbc.suite;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.jdbc2.JdbcBlobTest;
 import org.apache.ignite.internal.jdbc2.JdbcDistributedJoinsQueryTest;
 import org.apache.ignite.jdbc.JdbcComplexQuerySelfTest;
 import org.apache.ignite.jdbc.JdbcConnectionSelfTest;
@@ -42,7 +43,7 @@ public class IgniteJdbcDriverTestSuite extends TestSuite {
     public static TestSuite suite() throws Exception {
         TestSuite suite = new TestSuite("Ignite JDBC Driver Test Suite");
 
-        // Thin client based driver tests
+        // Thin client based driver tests.
         suite.addTest(new TestSuite(JdbcConnectionSelfTest.class));
         suite.addTest(new TestSuite(JdbcStatementSelfTest.class));
         suite.addTest(new TestSuite(JdbcPreparedStatementSelfTest.class));
@@ -61,7 +62,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcPreparedStatementSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcResultSetSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcComplexQuerySelfTest.class));
-        suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcDistributedJoinsQueryTest.class));
+        suite.addTest(new TestSuite(JdbcDistributedJoinsQueryTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcMetadataSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcEmptyCacheSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcLocalCachesSelfTest.class));
@@ -71,6 +72,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcInsertStatementSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcBinaryMarshallerInsertStatementSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcDeleteStatementSelfTest.class));
+        suite.addTest(new TestSuite(JdbcBlobTest.class));
 
         return suite;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBlob.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBlob.java
new file mode 100644
index 0000000000000..daa0d1fbd357d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBlob.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.sql.Blob;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.util.Arrays;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Simple BLOB implementation. Actually there is no such entity as BLOB in Ignite. So using arrays is preferable way
+ * to work with binary objects.
+ *
+ * This implementation can be useful for reading binary fields of objects through JDBC.
+ */
+public class JdbcBlob implements Blob {
+    /** Byte array. */
+    private byte[] arr;
+
+    /**
+     * @param arr Byte array.
+     */
+    public JdbcBlob(byte[] arr) {
+        this.arr = arr;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long length() throws SQLException {
+        ensureNotClosed();
+
+        return arr.length;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte[] getBytes(long pos, int len) throws SQLException {
+        ensureNotClosed();
+
+        if (pos < 1 || arr.length - pos < 0 || len < 0)
+            throw new SQLException("Invalid argument. Position can't be less than 1 or " +
+                "greater than size of underlying byte array. Requested length also can't be negative " + "" +
+                "[pos=" + pos + ", len=" + len +']');
+
+        int idx = (int)(pos - 1);
+
+        int size = len > arr.length - idx ? arr.length - idx : len;
+
+        byte[] res = new byte[size];
+
+        U.arrayCopy(arr, idx, res, 0, size);
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public InputStream getBinaryStream() throws SQLException {
+        ensureNotClosed();
+
+        return new ByteArrayInputStream(arr);
+    }
+
+    /** {@inheritDoc} */
+    @Override public InputStream getBinaryStream(long pos, long len) throws SQLException {
+        ensureNotClosed();
+
+        if (pos < 1 || len < 1 || pos > arr.length || len > arr.length - pos + 1)
+            throw new SQLException("Invalid argument. Position can't be less than 1 or " +
+                "greater than size of underlying byte array. Requested length can't be negative and can't be " +
+                "greater than available bytes from given position [pos=" + pos + ", len=" + len +']');
+
+
+        return new ByteArrayInputStream(arr, (int)(pos - 1), (int)len);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long position(byte[] ptrn, long start) throws SQLException {
+        ensureNotClosed();
+
+        if (start < 1 || start > arr.length || ptrn.length == 0 || ptrn.length > arr.length)
+            return -1;
+
+        for(int i = 0, pos = (int)(start - 1); pos < arr.length;) {
+            if (arr[pos] == ptrn[i]) {
+                pos++;
+
+                i++;
+
+                if (i == ptrn.length)
+                    return pos - ptrn.length + 1;
+            }
+            else {
+                pos = pos - i + 1;
+
+                i = 0;
+            }
+        }
+
+        return -1;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long position(Blob ptrn, long start) throws SQLException {
+        ensureNotClosed();
+
+        if (start < 1 || start > arr.length || ptrn.length() == 0 || ptrn.length() > arr.length)
+            return -1;
+
+        return position(ptrn.getBytes(1, (int)ptrn.length()), start);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int setBytes(long pos, byte[] bytes) throws SQLException {
+        return setBytes(pos, bytes, 0, bytes.length);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int setBytes(long pos, byte[] bytes, int off, int len) throws SQLException {
+        ensureNotClosed();
+
+        if (pos < 1)
+            throw new SQLException("Invalid argument. Position can't be less than 1 [pos=" + pos + ']');
+
+        int idx = (int)(pos - 1);
+
+        byte[] dst = arr;
+
+        if (idx + len > arr.length) {
+            dst = new byte[arr.length + (len - (arr.length - idx))];
+
+            U.arrayCopy(arr, 0, dst, 0, idx);
+
+            arr = dst;
+        }
+
+        U.arrayCopy(bytes, off, dst, idx, len);
+
+        return len;
+    }
+
+    /** {@inheritDoc} */
+    @Override public OutputStream setBinaryStream(long pos) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void truncate(long len) throws SQLException {
+        ensureNotClosed();
+
+        if (len < 0 || len > arr.length)
+            throw new SQLException("Invalid argument. Length can't be " +
+                "less than zero or greater than Blob length [len=" + len + ']');
+
+        arr = Arrays.copyOf(arr, (int)len);
+
+    }
+
+    /** {@inheritDoc} */
+    @Override public void free() throws SQLException {
+        if (arr != null)
+            arr = null;
+    }
+
+    /**
+     *
+     */
+    private void ensureNotClosed() throws SQLException {
+        if (arr == null)
+            throw new SQLException("Blob instance can't be used after free() has been called.");
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
index dc3fe7f454624..3810117a785ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
@@ -550,7 +550,7 @@ private IgniteConfiguration loadConfiguration(String cfgUrl) {
     @Override public Blob createBlob() throws SQLException {
         ensureNotClosed();
 
-        throw new SQLFeatureNotSupportedException("SQL-specific types are not supported.");
+        return new JdbcBlob(new byte[0]);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
index 57badd2df728e..371c009df0b8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
@@ -227,9 +227,7 @@ public class JdbcPreparedStatement extends JdbcStatement implements PreparedStat
 
     /** {@inheritDoc} */
     @Override public void setBlob(int paramIdx, Blob x) throws SQLException {
-        ensureNotClosed();
-
-        throw new SQLFeatureNotSupportedException("SQL-specific types are not supported.");
+        setBytes(paramIdx, x.getBytes(1, (int)x.length()));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
index b53521e4cfa4d..187930ec4cc66 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
@@ -1012,9 +1012,7 @@ void closeInternal() throws SQLException  {
 
     /** {@inheritDoc} */
     @Override public Blob getBlob(int colIdx) throws SQLException {
-        ensureNotClosed();
-
-        throw new SQLFeatureNotSupportedException("SQL-specific types are not supported.");
+        return new JdbcBlob(getBytes(colIdx));
     }
 
     /** {@inheritDoc} */
@@ -1045,9 +1043,7 @@ void closeInternal() throws SQLException  {
 
     /** {@inheritDoc} */
     @Override public Blob getBlob(String colLb) throws SQLException {
-        ensureNotClosed();
-
-        throw new SQLFeatureNotSupportedException("SQL-specific types are not supported.");
+        return new JdbcBlob(getBytes(colLb));
     }
 
     /** {@inheritDoc} */

From d77a134fffee431cd7fa0bae2349419bc97ec1cf Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 30 May 2017 19:00:47 +0300
Subject: [PATCH 179/516] IGNITE-5342 - Skip permission check for TASK_EXECUTE
 for service jobs

---
 .../processors/affinity/GridAffinityProcessor.java    |  2 +-
 .../internal/processors/cache/GridCacheAdapter.java   |  6 +++---
 .../datastructures/CacheDataStructuresManager.java    |  4 ++--
 .../processors/closure/GridClosureProcessor.java      | 11 ++++++++---
 .../processors/service/GridServiceProcessor.java      |  4 ++--
 .../internal/processors/service/GridServiceProxy.java |  4 ++--
 .../internal/processors/task/GridTaskProcessor.java   |  6 ++++--
 .../processors/task/GridTaskThreadContextKey.java     |  5 ++++-
 .../internal/IgniteComputeTopologyExceptionTest.java  |  2 +-
 9 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
index b6efafbab394c..1be59784db6ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
@@ -497,7 +497,7 @@ private <K> Map<ClusterNode, Collection<K>> keysToNodes(@Nullable final String c
     private AffinityInfo affinityInfoFromNode(@Nullable String cacheName, AffinityTopologyVersion topVer, ClusterNode n)
         throws IgniteCheckedException {
         GridTuple3<GridAffinityMessage, GridAffinityMessage, GridAffinityAssignment> t = ctx.closure()
-            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0).get();
+            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0, false).get();
 
         AffinityFunction f = (AffinityFunction)unmarshall(ctx, n.id(), t.get1());
         AffinityKeyMapper m = (AffinityKeyMapper)unmarshall(ctx, n.id(), t.get2());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 189f602328894..ac1d2682d5f2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -84,8 +84,8 @@
 import org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityImpl;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteExternalizableExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter;
@@ -3665,14 +3665,14 @@ private IgniteInternalFuture<?> runLoadKeysCallable(final Set<? extends K> keys,
                 new LoadKeysCallableV2<>(ctx.name(), keys, update, plc, keepBinary),
                 nodes,
                 true,
-                0);
+                0, false);
         }
         else {
             return ctx.closures().callAsyncNoFailover(BROADCAST,
                 new LoadKeysCallable<>(ctx.name(), keys, update, plc),
                 nodes,
                 true,
-                0);
+                0, false);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index c1983df4f4d86..366a4a920b9ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -492,7 +492,7 @@ public void removeSetData(IgniteUuid id) throws IgniteCheckedException {
                         new BlockSetCallable(cctx.name(), id),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
@@ -516,7 +516,7 @@ else if (!pingNodes(nodes)) {
                         new RemoveSetDataCallable(cctx.name(), id, topVer),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 20fb6a0d0f485..aea7fe0a1e89b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -80,6 +80,7 @@
 import static org.apache.ignite.compute.ComputeJobResultPolicy.FAILOVER;
 import static org.apache.ignite.compute.ComputeJobResultPolicy.REDUCE;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
 
@@ -514,11 +515,12 @@ public ComputeTaskInternalFuture<?> affinityRun(@NotNull Collection<String> cach
     }
 
     /**
+     * @param <R> Type.
      * @param mode Distribution mode.
      * @param job Closure to execute.
      * @param nodes Grid nodes.
      * @param sys If {@code true}, then system pool will be used.
-     * @param <R> Type.
+     * @param skipAuth Skip authorization check.
      * @return Grid future for collection of closure results.
      */
     public <R> IgniteInternalFuture<R> callAsyncNoFailover(
@@ -526,8 +528,8 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
         @Nullable Callable<R> job,
         @Nullable Collection<ClusterNode> nodes,
         boolean sys,
-        long timeout
-    ) {
+        long timeout,
+        boolean skipAuth) {
         assert mode != null;
         assert timeout >= 0 : timeout;
 
@@ -543,6 +545,9 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
             ctx.task().setThreadContext(TC_NO_FAILOVER, true);
             ctx.task().setThreadContext(TC_SUBGRID, nodes);
 
+            if (skipAuth)
+                ctx.task().setThreadContext(TC_SKIP_AUTH, true);
+
             if (timeout > 0)
                 ctx.task().setThreadContext(TC_TIMEOUT, timeout);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index d7b9abc7fc1bd..25a8edbfc7c17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -721,8 +721,8 @@ public Map<UUID, Integer> serviceTopology(String name, long timeout) throws Igni
                 call,
                 Collections.singletonList(node),
                 false,
-                timeout
-            ).get();
+                timeout,
+                true).get();
         }
         else
             return serviceTopology(cache, name);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index aa609340b3bfb..2286cff9717aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -182,8 +182,8 @@ else if (U.isToStringMethod(mtd))
                             new ServiceProxyCallable(mtd.getName(), name, mtd.getParameterTypes(), args),
                             Collections.singleton(node),
                             false,
-                            waitTimeout
-                        ).get();
+                            waitTimeout,
+                            true).get();
                     }
                 }
                 catch (GridServiceNotFoundException | ClusterTopologyCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 935686456e593..12213581b3286 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -82,6 +82,7 @@
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK;
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK_CANCEL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBJ_ID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TASK_NAME;
@@ -477,8 +478,6 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
         else
             taskClsName = taskCls != null ? taskCls.getName() : taskName;
 
-        ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
-
         // Get values from thread-local context.
         Map<GridTaskThreadContextKey, Object> map = thCtx.get();
 
@@ -488,6 +487,9 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
             // Reset thread-local context.
             thCtx.set(null);
 
+        if (map.get(TC_SKIP_AUTH) == null)
+            ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
+
         Long timeout = (Long)map.get(TC_TIMEOUT);
 
         long timeout0 = timeout == null || timeout == 0 ? Long.MAX_VALUE : timeout;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
index 3bb19241a4ebf..a45f851d902a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
@@ -34,5 +34,8 @@ public enum GridTaskThreadContextKey {
     TC_TIMEOUT,
 
     /** Security subject ID. */
-    TC_SUBJ_ID
+    TC_SUBJ_ID,
+
+    /** Skip authorization for the task. */
+    TC_SKIP_AUTH
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
index 3ed91e805d656..a82373b618507 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
@@ -89,7 +89,7 @@ public void testCorrectCheckedException() throws Exception {
             },
             nodes,
             false,
-            0);
+            0, false);
 
         try {
             fut.get();

From d0186c368b12ab6893d0985e3fb92600708d5b65 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 30 May 2017 19:00:47 +0300
Subject: [PATCH 180/516] IGNITE-5342 - Skip permission check for TASK_EXECUTE
 for service jobs

(cherry picked from commit d77a134)
---
 .../processors/affinity/GridAffinityProcessor.java    |  2 +-
 .../internal/processors/cache/GridCacheAdapter.java   |  5 ++---
 .../datastructures/CacheDataStructuresManager.java    |  4 ++--
 .../processors/closure/GridClosureProcessor.java      | 11 ++++++++---
 .../processors/service/GridServiceProcessor.java      |  4 ++--
 .../internal/processors/service/GridServiceProxy.java |  4 ++--
 .../internal/processors/task/GridTaskProcessor.java   |  6 ++++--
 .../processors/task/GridTaskThreadContextKey.java     |  5 ++++-
 .../internal/IgniteComputeTopologyExceptionTest.java  |  2 +-
 9 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
index b6efafbab394c..1be59784db6ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
@@ -497,7 +497,7 @@ private <K> Map<ClusterNode, Collection<K>> keysToNodes(@Nullable final String c
     private AffinityInfo affinityInfoFromNode(@Nullable String cacheName, AffinityTopologyVersion topVer, ClusterNode n)
         throws IgniteCheckedException {
         GridTuple3<GridAffinityMessage, GridAffinityMessage, GridAffinityAssignment> t = ctx.closure()
-            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0).get();
+            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0, false).get();
 
         AffinityFunction f = (AffinityFunction)unmarshall(ctx, n.id(), t.get1());
         AffinityKeyMapper m = (AffinityKeyMapper)unmarshall(ctx, n.id(), t.get2());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index f10406fce2a80..d84060db4ce58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -84,7 +84,6 @@
 import org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityImpl;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteExternalizableExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo;
@@ -3512,14 +3511,14 @@ private IgniteInternalFuture<?> runLoadKeysCallable(final Set<? extends K> keys,
                 new LoadKeysCallableV2<>(ctx.name(), keys, update, plc, keepBinary),
                 nodes,
                 true,
-                0);
+                0, false);
         }
         else {
             return ctx.closures().callAsyncNoFailover(BROADCAST,
                 new LoadKeysCallable<>(ctx.name(), keys, update, plc),
                 nodes,
                 true,
-                0);
+                0, false);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index c1983df4f4d86..366a4a920b9ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -492,7 +492,7 @@ public void removeSetData(IgniteUuid id) throws IgniteCheckedException {
                         new BlockSetCallable(cctx.name(), id),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
@@ -516,7 +516,7 @@ else if (!pingNodes(nodes)) {
                         new RemoveSetDataCallable(cctx.name(), id, topVer),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 20fb6a0d0f485..aea7fe0a1e89b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -80,6 +80,7 @@
 import static org.apache.ignite.compute.ComputeJobResultPolicy.FAILOVER;
 import static org.apache.ignite.compute.ComputeJobResultPolicy.REDUCE;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
 
@@ -514,11 +515,12 @@ public ComputeTaskInternalFuture<?> affinityRun(@NotNull Collection<String> cach
     }
 
     /**
+     * @param <R> Type.
      * @param mode Distribution mode.
      * @param job Closure to execute.
      * @param nodes Grid nodes.
      * @param sys If {@code true}, then system pool will be used.
-     * @param <R> Type.
+     * @param skipAuth Skip authorization check.
      * @return Grid future for collection of closure results.
      */
     public <R> IgniteInternalFuture<R> callAsyncNoFailover(
@@ -526,8 +528,8 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
         @Nullable Callable<R> job,
         @Nullable Collection<ClusterNode> nodes,
         boolean sys,
-        long timeout
-    ) {
+        long timeout,
+        boolean skipAuth) {
         assert mode != null;
         assert timeout >= 0 : timeout;
 
@@ -543,6 +545,9 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
             ctx.task().setThreadContext(TC_NO_FAILOVER, true);
             ctx.task().setThreadContext(TC_SUBGRID, nodes);
 
+            if (skipAuth)
+                ctx.task().setThreadContext(TC_SKIP_AUTH, true);
+
             if (timeout > 0)
                 ctx.task().setThreadContext(TC_TIMEOUT, timeout);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6bda8446b9b04..98439d4c4ec4b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -721,8 +721,8 @@ public Map<UUID, Integer> serviceTopology(String name, long timeout) throws Igni
                 call,
                 Collections.singletonList(node),
                 false,
-                timeout
-            ).get();
+                timeout,
+                true).get();
         }
         else
             return serviceTopology(cache, name);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index d2e96bac52c0c..d16a4c48dcf86 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -192,8 +192,8 @@ else if (U.isToStringMethod(mtd))
                             new ServiceProxyCallable(mtd.getName(), name, mtd.getParameterTypes(), args),
                             Collections.singleton(node),
                             false,
-                            waitTimeout
-                        ).get();
+                            waitTimeout,
+                            true).get();
                     }
                 }
                 catch (GridServiceNotFoundException | ClusterTopologyCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 935686456e593..12213581b3286 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -82,6 +82,7 @@
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK;
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK_CANCEL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBJ_ID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TASK_NAME;
@@ -477,8 +478,6 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
         else
             taskClsName = taskCls != null ? taskCls.getName() : taskName;
 
-        ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
-
         // Get values from thread-local context.
         Map<GridTaskThreadContextKey, Object> map = thCtx.get();
 
@@ -488,6 +487,9 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
             // Reset thread-local context.
             thCtx.set(null);
 
+        if (map.get(TC_SKIP_AUTH) == null)
+            ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
+
         Long timeout = (Long)map.get(TC_TIMEOUT);
 
         long timeout0 = timeout == null || timeout == 0 ? Long.MAX_VALUE : timeout;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
index 2ec63df4ed83c..f0e56c731fd2c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
@@ -37,5 +37,8 @@ public enum GridTaskThreadContextKey {
     TC_SUBJ_ID,
 
     /** IO manager policy. */
-    TC_IO_POLICY
+    TC_IO_POLICY,
+
+    /** Skip authorization for the task. */
+    TC_SKIP_AUTH
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
index 3ed91e805d656..a82373b618507 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
@@ -89,7 +89,7 @@ public void testCorrectCheckedException() throws Exception {
             },
             nodes,
             false,
-            0);
+            0, false);
 
         try {
             fut.get();

From 0a0c5be1eb4793bc838da3184591d91537bd9cad Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 31 May 2017 15:24:14 +0300
Subject: [PATCH 181/516] Merge compilation fix.

---
 .../apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
index 0b3f2e318e3d5..9c72da5a76077 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.binary.BinaryFooterOffsetsHeapSelfTest;
 import org.apache.ignite.internal.binary.BinaryFooterOffsetsOffheapSelfTest;
 import org.apache.ignite.internal.binary.BinaryIdentityResolverConfigurationSelfTest;
+import org.apache.ignite.internal.binary.BinaryMarshallerLocalMetadataCacheTest;
 import org.apache.ignite.internal.binary.BinaryMarshallerSelfTest;
 import org.apache.ignite.internal.binary.BinaryObjectBuilderAdditionalSelfTest;
 import org.apache.ignite.internal.binary.BinaryObjectBuilderDefaultMappersSelfTest;

From bc6538e5052ff0109fa8a57e51e4678a74e89ad3 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 30 May 2017 19:00:47 +0300
Subject: [PATCH 182/516] IGNITE-5342 - Skip permission check for TASK_EXECUTE
 for service jobs

(cherry picked from commit d77a134)
---
 .../processors/affinity/GridAffinityProcessor.java    |  2 +-
 .../internal/processors/cache/GridCacheAdapter.java   |  4 ++--
 .../datastructures/CacheDataStructuresManager.java    |  4 ++--
 .../processors/closure/GridClosureProcessor.java      | 11 ++++++++---
 .../processors/service/GridServiceProcessor.java      |  4 ++--
 .../internal/processors/service/GridServiceProxy.java |  4 ++--
 .../internal/processors/task/GridTaskProcessor.java   |  6 ++++--
 .../processors/task/GridTaskThreadContextKey.java     |  5 ++++-
 .../internal/IgniteComputeTopologyExceptionTest.java  |  2 +-
 9 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
index b6efafbab394c..1be59784db6ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
@@ -497,7 +497,7 @@ private <K> Map<ClusterNode, Collection<K>> keysToNodes(@Nullable final String c
     private AffinityInfo affinityInfoFromNode(@Nullable String cacheName, AffinityTopologyVersion topVer, ClusterNode n)
         throws IgniteCheckedException {
         GridTuple3<GridAffinityMessage, GridAffinityMessage, GridAffinityAssignment> t = ctx.closure()
-            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0).get();
+            .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0, false).get();
 
         AffinityFunction f = (AffinityFunction)unmarshall(ctx, n.id(), t.get1());
         AffinityKeyMapper m = (AffinityKeyMapper)unmarshall(ctx, n.id(), t.get2());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 67ae9608b0f65..e17bf08dd3db6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3555,14 +3555,14 @@ private IgniteInternalFuture<?> runLoadKeysCallable(final Set<? extends K> keys,
                 new LoadKeysCallableV2<>(ctx.name(), keys, update, plc, keepBinary),
                 nodes,
                 true,
-                0);
+                0, false);
         }
         else {
             return ctx.closures().callAsyncNoFailover(BROADCAST,
                 new LoadKeysCallable<>(ctx.name(), keys, update, plc),
                 nodes,
                 true,
-                0);
+                0, false);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index d864d3c01b988..2b3080981ec36 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -492,7 +492,7 @@ public void removeSetData(IgniteUuid id) throws IgniteCheckedException {
                         new BlockSetCallable(cctx.name(), id),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
@@ -516,7 +516,7 @@ else if (!pingNodes(nodes)) {
                         new RemoveSetDataCallable(cctx.name(), id, topVer),
                         nodes,
                         true,
-                        0).get();
+                        0, false).get();
                 }
                 catch (IgniteCheckedException e) {
                     if (e.hasCause(ClusterTopologyCheckedException.class)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 61ed8a0b9e929..31cf0393e056d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -80,6 +80,7 @@
 import static org.apache.ignite.compute.ComputeJobResultPolicy.FAILOVER;
 import static org.apache.ignite.compute.ComputeJobResultPolicy.REDUCE;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
 
@@ -514,11 +515,12 @@ public ComputeTaskInternalFuture<?> affinityRun(@NotNull Collection<String> cach
     }
 
     /**
+     * @param <R> Type.
      * @param mode Distribution mode.
      * @param job Closure to execute.
      * @param nodes Grid nodes.
      * @param sys If {@code true}, then system pool will be used.
-     * @param <R> Type.
+     * @param skipAuth Skip authorization check.
      * @return Grid future for collection of closure results.
      */
     public <R> IgniteInternalFuture<R> callAsyncNoFailover(
@@ -526,8 +528,8 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
         @Nullable Callable<R> job,
         @Nullable Collection<ClusterNode> nodes,
         boolean sys,
-        long timeout
-    ) {
+        long timeout,
+        boolean skipAuth) {
         assert mode != null;
         assert timeout >= 0 : timeout;
 
@@ -543,6 +545,9 @@ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
             ctx.task().setThreadContext(TC_NO_FAILOVER, true);
             ctx.task().setThreadContext(TC_SUBGRID, nodes);
 
+            if (skipAuth)
+                ctx.task().setThreadContext(TC_SKIP_AUTH, true);
+
             if (timeout > 0)
                 ctx.task().setThreadContext(TC_TIMEOUT, timeout);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6bda8446b9b04..98439d4c4ec4b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -721,8 +721,8 @@ public Map<UUID, Integer> serviceTopology(String name, long timeout) throws Igni
                 call,
                 Collections.singletonList(node),
                 false,
-                timeout
-            ).get();
+                timeout,
+                true).get();
         }
         else
             return serviceTopology(cache, name);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index d2e96bac52c0c..d16a4c48dcf86 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -192,8 +192,8 @@ else if (U.isToStringMethod(mtd))
                             new ServiceProxyCallable(mtd.getName(), name, mtd.getParameterTypes(), args),
                             Collections.singleton(node),
                             false,
-                            waitTimeout
-                        ).get();
+                            waitTimeout,
+                            true).get();
                     }
                 }
                 catch (GridServiceNotFoundException | ClusterTopologyCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 935686456e593..12213581b3286 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -82,6 +82,7 @@
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK;
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK_CANCEL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBJ_ID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TASK_NAME;
@@ -477,8 +478,6 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
         else
             taskClsName = taskCls != null ? taskCls.getName() : taskName;
 
-        ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
-
         // Get values from thread-local context.
         Map<GridTaskThreadContextKey, Object> map = thCtx.get();
 
@@ -488,6 +487,9 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
             // Reset thread-local context.
             thCtx.set(null);
 
+        if (map.get(TC_SKIP_AUTH) == null)
+            ctx.security().authorize(taskClsName, SecurityPermission.TASK_EXECUTE, null);
+
         Long timeout = (Long)map.get(TC_TIMEOUT);
 
         long timeout0 = timeout == null || timeout == 0 ? Long.MAX_VALUE : timeout;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
index 2ec63df4ed83c..f0e56c731fd2c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
@@ -37,5 +37,8 @@ public enum GridTaskThreadContextKey {
     TC_SUBJ_ID,
 
     /** IO manager policy. */
-    TC_IO_POLICY
+    TC_IO_POLICY,
+
+    /** Skip authorization for the task. */
+    TC_SKIP_AUTH
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
index 3ed91e805d656..a82373b618507 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
@@ -89,7 +89,7 @@ public void testCorrectCheckedException() throws Exception {
             },
             nodes,
             false,
-            0);
+            0, false);
 
         try {
             fut.get();

From b52ea9299a007b0e5b7c0724e7756e012a6232ea Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 31 May 2017 15:35:34 +0300
Subject: [PATCH 183/516] IGNITE-5210 - If enabled security authentication,
 server is unable to restart if client tries to reconnect

(cherry picked from commit a7e5660)
---
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 12 ++-
 .../discovery/AuthenticationRestartTest.java  | 88 +++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |  2 +
 3 files changed, 99 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index dcda742a764a2..a5a4e88990c4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -634,9 +634,12 @@ else if (addrs.isEmpty()) {
                 if (!recon) {
                     TcpDiscoveryNode node = locNode;
 
-                    if (locNode.order() > 0)
+                    if (locNode.order() > 0) {
                         node = locNode.clientReconnectNode(spi.spiCtx.nodeAttributes());
 
+                        marshalCredentials(node);
+                    }
+
                     msg = new TcpDiscoveryJoinRequestMessage(node, spi.collectExchangeData(getLocalNodeId()));
                 }
                 else
@@ -729,8 +732,11 @@ private void marshalCredentials(TcpDiscoveryNode node) throws IgniteSpiException
             // Use security-unsafe getter.
             Map<String, Object> attrs = new HashMap<>(node.getAttributes());
 
-            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS,
-                U.marshal(spi.marshaller(), attrs.get(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS)));
+            Object creds = attrs.get(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS);
+
+            assert !(creds instanceof byte[]);
+
+            attrs.put(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS, U.marshal(spi.marshaller(), creds));
 
             node.setAttributes(attrs);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java
new file mode 100644
index 0000000000000..5841094bf2749
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/AuthenticationRestartTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TestReconnectPluginProvider;
+import org.apache.ignite.spi.discovery.tcp.TestReconnectProcessor;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/**
+ * Checks whether client is able to reconnect to restarted cluster with
+ * enabled security.
+ */
+public class AuthenticationRestartTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setClientMode(igniteInstanceName.contains("client"));
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setJoinTimeout(1120_000);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        TestReconnectPluginProvider.enabled = true;
+        TestReconnectProcessor.enabled = true;
+
+        startGrid("server");
+        startGrid("client");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        TestReconnectPluginProvider.enabled = false;
+        TestReconnectProcessor.enabled = false;
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientReconnect() throws Exception {
+        stopGrid("server");
+
+        final IgniteEx client = grid("client");
+
+        waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return client.cluster().clientReconnectFuture() != null;
+            }
+        }, 10_000);
+
+        startGrid("server");
+
+        IgniteFuture<?> fut = client.cluster().clientReconnectFuture();
+
+        assertNotNull(fut);
+
+        fut.get();
+
+        assertEquals(2, client.cluster().nodes().size());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index e6b39f740a0dd..12871492d0979 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.spi.GridTcpSpiForwardingSelfTest;
+import org.apache.ignite.spi.discovery.AuthenticationRestartTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest;
@@ -87,6 +88,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(TcpDiscoveryMultiThreadedTest.class));
 
         suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
+        suite.addTest(new TestSuite(AuthenticationRestartTest.class));
 
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));

From 10edc63b624685cf13c932497e38dad6f7a30a9f Mon Sep 17 00:00:00 2001
From: Sergi Vladykin <sergi.vladykin@gmail.com>
Date: Tue, 9 May 2017 18:17:40 +0300
Subject: [PATCH 184/516] master - minor fix for subqueries with aggregates

---
 .../internal/processors/query/h2/sql/GridSqlQuerySplitter.java  | 2 +-
 .../internal/processors/query/IgniteSqlSplitterSelfTest.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 5bd7f2b9772b6..35b30055795c2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -1760,7 +1760,7 @@ private static boolean hasAggregates(GridSqlAst el) {
 
         // If in SELECT clause we have a subquery expression with aggregate,
         // we should not split it. Run the whole subquery on MAP stage.
-        if (el instanceof GridSqlQuery)
+        if (el instanceof GridSqlSubquery)
             return false;
 
         for (int i = 0; i < el.size(); i++) {
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 8daf97d50b69a..5144755b9aa70 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -198,7 +198,7 @@ public void testSubQueryWithAggregate() {
             c1.put(new AffinityKey<>(2, orgId), new Person2(orgId, "Another Vasya"));
 
             List<List<?>> rs = c1.query(new SqlFieldsQuery("select name, " +
-                "(select count(1) from Person2 q where q.orgId = p.orgId) " +
+                "select count(1) from Person2 q where q.orgId = p.orgId " +
                 "from Person2 p order by name desc")).getAll();
 
             assertEquals(2, rs.size());

From 28a411653e826cebba5c2c396a6be681e36f08e4 Mon Sep 17 00:00:00 2001
From: Sergi Vladykin <sergi.vladykin@gmail.com>
Date: Thu, 11 May 2017 16:01:38 +0300
Subject: [PATCH 185/516] IGNITE-5190 - ArrayIndexOutOfBoundsException in
 GridMergeIndexSorted

---
 .../h2/twostep/GridMergeIndexSorted.java      |  3 +
 .../query/IgniteSqlSplitterSelfTest.java      | 68 +++++++++++++++++++
 2 files changed, 71 insertions(+)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
index 361bb2d5878c0..ce01fefec9b27 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
@@ -250,6 +250,9 @@ private void goFirst() {
          *
          */
         private void goNext() {
+            if (off == streams.length)
+                return; // All streams are done.
+
             if (streams[off].next())
                 bubbleUp(streams, off, streamCmp);
             else
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 5144755b9aa70..b9fc22f5c617d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -1424,6 +1424,37 @@ public void testImplicitJoinConditionGeneration() {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testJoinWithSubquery() throws Exception {
+        IgniteCache<Integer, Contract> c1 = ignite(0).createCache(
+            cacheConfig("Contract", true,
+            Integer.class, Contract.class));
+
+        IgniteCache<Integer, PromoContract> c2 = ignite(0).createCache(
+            cacheConfig("PromoContract", true,
+            Integer.class, PromoContract.class));
+
+        for (int i = 0; i < 100; i++) {
+            int coId = i % 10;
+            int cust = i / 10;
+            c1.put( i, new Contract(coId, cust));
+        }
+
+        for (int i = 0; i < 10; i++)
+            c2.put(i, new PromoContract((i % 5) + 1, i));
+
+        final List<List<?>> res = c2.query(new SqlFieldsQuery("SELECT CO.CO_ID \n" +
+            "FROM PromoContract PMC  \n" +
+            "INNER JOIN \"Contract\".Contract CO  ON PMC.CO_ID = 5  \n" +
+            "AND PMC.CO_ID = CO.CO_ID  \n" +
+            "INNER JOIN  (SELECT CO_ID FROM PromoContract EBP WHERE EBP.CO_ID = 5 LIMIT 1) VPMC  \n" +
+            "ON PMC.CO_ID = VPMC.CO_ID ")).getAll();
+
+        assertFalse(res.isEmpty());
+    }
+
     /** @throws Exception if failed. */
     public void testDistributedAggregates() throws Exception {
         final String cacheName = "ints";
@@ -1943,4 +1974,41 @@ private static class OrderGood implements Serializable {
         @QuerySqlField
         private int goodId;
     }
+
+    /** */
+    private static class Contract implements Serializable {
+        /** */
+        @QuerySqlField(index = true)
+        private final int CO_ID;
+
+        /** */
+        @QuerySqlField(index = true)
+        private final int CUSTOMER_ID;
+
+        /** */
+        public Contract(final int CO_ID, final int CUSTOMER_ID) {
+            this.CO_ID = CO_ID;
+            this.CUSTOMER_ID = CUSTOMER_ID;
+        }
+
+    }
+
+    /** */
+    public class PromoContract implements Serializable {
+        /** */
+        @QuerySqlField(index = true, orderedGroups = {
+            @QuerySqlField.Group(name = "myIdx", order = 1)})
+        private final int CO_ID;
+
+        /** */
+        @QuerySqlField(index = true, orderedGroups = {
+            @QuerySqlField.Group(name = "myIdx", order = 0)})
+        private final int OFFER_ID;
+
+        /** */
+        public PromoContract(final int co_Id, final int offer_Id) {
+            this.CO_ID = co_Id;
+            this.OFFER_ID = offer_Id;
+        }
+    }
 }

From 950a07b48d0d1f7203834fff3306018cf4ff0cf4 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 1 Jun 2017 14:17:52 +0300
Subject: [PATCH 186/516] Minor fix

---
 .../internal/processors/query/h2/opt/GridH2IndexBase.java      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index efa95a8979b34..1389460489f8f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -316,7 +316,8 @@ protected <T> T threadLocalSnapshot() {
     public void releaseSnapshot() {
         Object s = snapshot.get();
 
-        assert s != null;
+        if (s == null)
+            return; // Nothing to do.
 
         snapshot.remove();
 

From abf4f325217c27cb5399aedc517e763092174d79 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 1 Jun 2017 14:49:20 +0300
Subject: [PATCH 187/516] Revert snapshot fix and fix tests.

---
 .../query/h2/opt/GridH2IndexBase.java         |   3 +-
 .../query/h2/opt/GridH2TableSelfTest.java     | 324 ++++++++++--------
 2 files changed, 184 insertions(+), 143 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 1389460489f8f..efa95a8979b34 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -316,8 +316,7 @@ protected <T> T threadLocalSnapshot() {
     public void releaseSnapshot() {
         Object s = snapshot.get();
 
-        if (s == null)
-            return; // Nothing to do.
+        assert s != null;
 
         snapshot.remove();
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TableSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TableSelfTest.java
index 8408ba06839ac..21515f9b2208e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TableSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TableSelfTest.java
@@ -128,7 +128,6 @@ private GridH2Row row(UUID id, long t, String str, long x) {
             ValueLong.get(x));
     }
 
-
     /**
      * Simple table test.
      *
@@ -140,137 +139,147 @@ public void testTable() throws Exception {
 
         Random rnd = new Random();
 
-        while(x-- > 0) {
-            UUID id = UUID.randomUUID();
+        GridH2QueryContext qctx = new GridH2QueryContext(UUID.randomUUID(), UUID.randomUUID(),
+            Thread.currentThread().getId(), GridH2QueryType.LOCAL);
 
-            GridH2Row row = row(id, System.currentTimeMillis(), rnd.nextBoolean() ? id.toString() :
-                UUID.randomUUID().toString(), rnd.nextInt(100));
+        try {
+            GridH2QueryContext.set(qctx);
 
-            tbl.doUpdate(row, false);
-        }
+            while (x-- > 0) {
+                UUID id = UUID.randomUUID();
 
-        assertEquals(MAX_X, tbl.getRowCountApproximation());
-        assertEquals(MAX_X, tbl.getRowCount(null));
+                GridH2Row row = row(id, System.currentTimeMillis(), rnd.nextBoolean() ? id.toString() :
+                    UUID.randomUUID().toString(), rnd.nextInt(100));
 
-        for (GridH2IndexBase idx : tbl.indexes()) {
-            assertEquals(MAX_X, idx.getRowCountApproximation());
-            assertEquals(MAX_X, idx.getRowCount(null));
-        }
+                tbl.doUpdate(row, false);
+            }
 
-        // Check correct rows order.
-        checkOrdered((GridH2TreeIndex)tbl.indexes().get(0), new Comparator<SearchRow>() {
-            @Override public int compare(SearchRow o1, SearchRow o2) {
-                UUID id1 = (UUID)o1.getValue(0).getObject();
-                UUID id2 = (UUID)o2.getValue(0).getObject();
+            assertEquals(MAX_X, tbl.getRowCountApproximation());
+            assertEquals(MAX_X, tbl.getRowCount(null));
 
-                return id1.compareTo(id2);
+            for (GridH2IndexBase idx : tbl.indexes()) {
+                assertEquals(MAX_X, idx.getRowCountApproximation());
+                assertEquals(MAX_X, idx.getRowCount(null));
             }
-        });
 
-        checkOrdered((GridH2TreeIndex)tbl.indexes().get(1), new Comparator<SearchRow>() {
-            @Override public int compare(SearchRow o1, SearchRow o2) {
-                Long x1 = (Long)o1.getValue(3).getObject();
-                Long x2 = (Long)o2.getValue(3).getObject();
+            // Check correct rows order.
+            checkOrdered((GridH2TreeIndex)tbl.indexes().get(0), new Comparator<SearchRow>() {
+                @Override public int compare(SearchRow o1, SearchRow o2) {
+                    UUID id1 = (UUID)o1.getValue(0).getObject();
+                    UUID id2 = (UUID)o2.getValue(0).getObject();
 
-                int c = x2.compareTo(x1);
+                    return id1.compareTo(id2);
+                }
+            });
 
-                if (c != 0)
-                    return c;
+            checkOrdered((GridH2TreeIndex)tbl.indexes().get(1), new Comparator<SearchRow>() {
+                @Override public int compare(SearchRow o1, SearchRow o2) {
+                    Long x1 = (Long)o1.getValue(3).getObject();
+                    Long x2 = (Long)o2.getValue(3).getObject();
 
-                Timestamp t1 = (Timestamp)o1.getValue(1).getObject();
-                Timestamp t2 = (Timestamp)o2.getValue(1).getObject();
+                    int c = x2.compareTo(x1);
 
-                return t1.compareTo(t2);
-            }
-        });
+                    if (c != 0)
+                        return c;
 
-        checkOrdered((GridH2TreeIndex)tbl.indexes().get(2), new Comparator<SearchRow>() {
-            @Override public int compare(SearchRow o1, SearchRow o2) {
-                String s1 = (String)o1.getValue(2).getObject();
-                String s2 = (String)o2.getValue(2).getObject();
+                    Timestamp t1 = (Timestamp)o1.getValue(1).getObject();
+                    Timestamp t2 = (Timestamp)o2.getValue(1).getObject();
 
-                return s2.compareTo(s1);
-            }
-        });
+                    return t1.compareTo(t2);
+                }
+            });
 
-        // Indexes data consistency.
-        ArrayList<? extends Index> idxs = tbl.indexes();
+            checkOrdered((GridH2TreeIndex)tbl.indexes().get(2), new Comparator<SearchRow>() {
+                @Override public int compare(SearchRow o1, SearchRow o2) {
+                    String s1 = (String)o1.getValue(2).getObject();
+                    String s2 = (String)o2.getValue(2).getObject();
 
-        checkIndexesConsistent((ArrayList<Index>)idxs, null);
+                    return s2.compareTo(s1);
+                }
+            });
 
-        // Check unique index.
-        UUID id = UUID.randomUUID();
-        UUID id2 = UUID.randomUUID();
+            // Indexes data consistency.
+            ArrayList<? extends Index> idxs = tbl.indexes();
 
-        assertTrue(tbl.doUpdate(row(id, System.currentTimeMillis(), id.toString(), rnd.nextInt(100)), false));
-        assertTrue(tbl.doUpdate(row(id2, System.currentTimeMillis(), id2.toString(), rnd.nextInt(100)), false));
+            checkIndexesConsistent((ArrayList<Index>)idxs, null);
 
-        // Check index selection.
-        checkQueryPlan(conn, "SELECT * FROM T", SCAN_IDX_NAME);
+            // Check unique index.
+            UUID id = UUID.randomUUID();
+            UUID id2 = UUID.randomUUID();
 
-        checkQueryPlan(conn, "SELECT * FROM T WHERE ID IS NULL", PK_NAME);
-        checkQueryPlan(conn, "SELECT * FROM T WHERE ID = RANDOM_UUID()", PK_NAME);
-        checkQueryPlan(conn, "SELECT * FROM T WHERE ID > RANDOM_UUID()", PK_NAME);
-        checkQueryPlan(conn, "SELECT * FROM T ORDER BY ID", PK_NAME);
+            assertTrue(tbl.doUpdate(row(id, System.currentTimeMillis(), id.toString(), rnd.nextInt(100)), false));
+            assertTrue(tbl.doUpdate(row(id2, System.currentTimeMillis(), id2.toString(), rnd.nextInt(100)), false));
 
-        checkQueryPlan(conn, "SELECT * FROM T WHERE STR IS NULL", STR_IDX_NAME);
-        checkQueryPlan(conn, "SELECT * FROM T WHERE STR = 'aaaa'", STR_IDX_NAME);
-        checkQueryPlan(conn, "SELECT * FROM T WHERE STR > 'aaaa'", STR_IDX_NAME);
-        checkQueryPlan(conn, "SELECT * FROM T ORDER BY STR DESC", STR_IDX_NAME);
+            // Check index selection.
+            checkQueryPlan(conn, "SELECT * FROM T", SCAN_IDX_NAME);
 
-        checkQueryPlan(conn, "SELECT * FROM T WHERE X IS NULL", NON_UNIQUE_IDX_NAME);
-        checkQueryPlan(conn, "SELECT * FROM T WHERE X = 10000", NON_UNIQUE_IDX_NAME);
-        checkQueryPlan(conn, "SELECT * FROM T WHERE X > 10000", NON_UNIQUE_IDX_NAME);
-        checkQueryPlan(conn, "SELECT * FROM T ORDER BY X DESC", NON_UNIQUE_IDX_NAME);
-        checkQueryPlan(conn, "SELECT * FROM T ORDER BY X DESC, T", NON_UNIQUE_IDX_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T WHERE ID IS NULL", PK_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T WHERE ID = RANDOM_UUID()", PK_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T WHERE ID > RANDOM_UUID()", PK_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T ORDER BY ID", PK_NAME);
 
-        checkQueryPlan(conn, "SELECT * FROM T ORDER BY T, X DESC", SCAN_IDX_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T WHERE STR IS NULL", STR_IDX_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T WHERE STR = 'aaaa'", STR_IDX_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T WHERE STR > 'aaaa'", STR_IDX_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T ORDER BY STR DESC", STR_IDX_NAME);
 
-        // Simple queries.
+            checkQueryPlan(conn, "SELECT * FROM T WHERE X IS NULL", NON_UNIQUE_IDX_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T WHERE X = 10000", NON_UNIQUE_IDX_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T WHERE X > 10000", NON_UNIQUE_IDX_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T ORDER BY X DESC", NON_UNIQUE_IDX_NAME);
+            checkQueryPlan(conn, "SELECT * FROM T ORDER BY X DESC, T", NON_UNIQUE_IDX_NAME);
 
-        Statement s = conn.createStatement();
+            checkQueryPlan(conn, "SELECT * FROM T ORDER BY T, X DESC", SCAN_IDX_NAME);
 
-        ResultSet rs = s.executeQuery("select id from t where x between 0 and 100");
+            // Simple queries.
 
-        int i = 0;
-        while (rs.next())
-            i++;
+            Statement s = conn.createStatement();
 
-        assertEquals(MAX_X + 2, i);
+            ResultSet rs = s.executeQuery("select id from t where x between 0 and 100");
 
-        // -----
+            int i = 0;
+            while (rs.next())
+                i++;
 
-        rs = s.executeQuery("select id from t where t is not null");
+            assertEquals(MAX_X + 2, i);
 
-        i = 0;
-        while (rs.next())
-            i++;
+            // -----
 
-        assertEquals(MAX_X + 2, i);
+            rs = s.executeQuery("select id from t where t is not null");
 
-        // ----
+            i = 0;
+            while (rs.next())
+                i++;
 
-        int cnt = 10 + rnd.nextInt(25);
+            assertEquals(MAX_X + 2, i);
 
-        long t = System.currentTimeMillis();
+            // ----
 
-        for (i = 0; i < cnt; i++) {
-            id = UUID.randomUUID();
+            int cnt = 10 + rnd.nextInt(25);
 
-            assertTrue(tbl.doUpdate(row(id, t, id.toString(), 51), false));
-        }
+            long t = System.currentTimeMillis();
 
-        rs = s.executeQuery("select x, id from t where x = 51 limit " + cnt);
+            for (i = 0; i < cnt; i++) {
+                id = UUID.randomUUID();
 
-        i = 0;
+                assertTrue(tbl.doUpdate(row(id, t, id.toString(), 51), false));
+            }
 
-        while (rs.next()) {
-            assertEquals(51, rs.getInt(1));
+            rs = s.executeQuery("select x, id from t where x = 51 limit " + cnt);
 
-            i++;
-        }
+            i = 0;
+
+            while (rs.next()) {
+                assertEquals(51, rs.getInt(1));
+
+                i++;
+            }
 
-        assertEquals(cnt, i);
+            assertEquals(cnt, i);
+        }
+        finally {
+            qctx.clearContext(true);
+        }
     }
 
     /**
@@ -296,6 +305,9 @@ public void testIndexesMultiThreadedConsistency() throws Exception {
 
         multithreaded(new Callable<Void>() {
             @Override public Void call() throws Exception {
+                GridH2QueryContext.set(new GridH2QueryContext(UUID.randomUUID(), UUID.randomUUID(),
+                    Thread.currentThread().getId(), GridH2QueryType.LOCAL));
+
                 Random rnd = new Random();
 
                 PreparedStatement ps1 = null;
@@ -361,7 +373,7 @@ public void testIndexesMultiThreadedConsistency() throws Exception {
 
                         rs = ps1.executeQuery();
 
-                        for (;;) {
+                        for (; ; ) {
                             assertTrue(rs.next());
 
                             if (rs.getObject(1).equals(id))
@@ -383,8 +395,8 @@ public void testIndexesMultiThreadedConsistency() throws Exception {
      * @throws Exception If failed.
      */
     @SuppressWarnings("InfiniteLoopStatement")
-    public static void main(String ... args) throws Exception {
-        for (int i = 0;;) {
+    public static void main(String... args) throws Exception {
+        for (int i = 0; ; ) {
             GridH2TableSelfTest t = new GridH2TableSelfTest();
 
             t.beforeTest();
@@ -398,7 +410,7 @@ public static void main(String ... args) throws Exception {
     }
 
     /**
-      * @throws Exception If failed.
+     * @throws Exception If failed.
      */
     public void testRangeQuery() throws Exception {
         int rows = 3000;
@@ -408,29 +420,39 @@ public void testRangeQuery() throws Exception {
 
         Random rnd = new Random();
 
-        for (int i = 0 ; i < rows; i++) {
-            UUID id = UUID.randomUUID();
+        GridH2QueryContext qctx = new GridH2QueryContext(UUID.randomUUID(), UUID.randomUUID(),
+            Thread.currentThread().getId(), GridH2QueryType.LOCAL);
 
-            GridH2Row row = row(id, t++, id.toString(), rnd.nextInt(xs));
+        try {
+            GridH2QueryContext.set(qctx);
 
-            assertTrue(tbl.doUpdate(row, false));
-        }
+            for (int i = 0; i < rows; i++) {
+                UUID id = UUID.randomUUID();
 
-        PreparedStatement ps = conn.prepareStatement("select count(*) from t where x = ?");
+                GridH2Row row = row(id, t++, id.toString(), rnd.nextInt(xs));
 
-        int cnt = 0;
+                assertTrue(tbl.doUpdate(row, false));
+            }
 
-        for (int x = 0; x < xs; x++) {
-            ps.setInt(1, x);
+            PreparedStatement ps = conn.prepareStatement("select count(*) from t where x = ?");
 
-            ResultSet rs = ps.executeQuery();
+            int cnt = 0;
 
-            assertTrue(rs.next());
+            for (int x = 0; x < xs; x++) {
+                ps.setInt(1, x);
 
-            cnt += rs.getInt(1);
-        }
+                ResultSet rs = ps.executeQuery();
+
+                assertTrue(rs.next());
 
-        assertEquals(rows, cnt);
+                cnt += rs.getInt(1);
+            }
+
+            assertEquals(rows, cnt);
+        }
+        finally {
+            qctx.clearContext(false);
+        }
     }
 
     /**
@@ -453,62 +475,82 @@ public void testDataLoss() throws Exception {
 
         multithreaded(new Callable<Void>() {
             @Override public Void call() throws Exception {
-                Random rnd = new Random();
+                GridH2QueryContext qctx = new GridH2QueryContext(UUID.randomUUID(), UUID.randomUUID(),
+                    Thread.currentThread().getId(), GridH2QueryType.LOCAL);
+
+                try {
+                    GridH2QueryContext.set(qctx);
+
+                    Random rnd = new Random();
 
-                int offset = cntr.getAndIncrement() * iterations;
+                    int offset = cntr.getAndIncrement() * iterations;
 
-                synchronized (ids[offset]) {
-                    for (int i = 0; i < iterations; i++) {
-                        UUID id = ids[offset + i];
+                    synchronized (ids[offset]) {
+                        for (int i = 0; i < iterations; i++) {
+                            UUID id = ids[offset + i];
 
-                        int x = rnd.nextInt(50);
+                            int x = rnd.nextInt(50);
 
-                        GridH2Row row = row(id, t, id.toString(), x);
+                            GridH2Row row = row(id, t, id.toString(), x);
 
-                        assertTrue(tbl.doUpdate(row, false));
+                            assertTrue(tbl.doUpdate(row, false));
+                        }
                     }
-                }
 
-                offset = (offset + iterations) % ids.length;
+                    offset = (offset + iterations) % ids.length;
 
-                synchronized (ids[offset]) {
-                    for (int i = 0; i < iterations; i += 2) {
-                        UUID id = ids[offset + i];
+                    synchronized (ids[offset]) {
+                        for (int i = 0; i < iterations; i += 2) {
+                            UUID id = ids[offset + i];
 
-                        int x = rnd.nextInt(50);
+                            int x = rnd.nextInt(50);
 
-                        GridH2Row row = row(id, t, id.toString(), x);
+                            GridH2Row row = row(id, t, id.toString(), x);
 
-                        if (tbl.doUpdate(row, true))
-                            deleted.incrementAndGet();
+                            if (tbl.doUpdate(row, true))
+                                deleted.incrementAndGet();
+                        }
                     }
-                }
 
-                return null;
+                    return null;
+                }
+                finally {
+                    qctx.clearContext(false);
+                }
             }
         }, threads);
 
         assertTrue(deleted.get() > 0);
 
-        PreparedStatement p = conn.prepareStatement("select count(*) from t where id = ?");
+        GridH2QueryContext qctx = new GridH2QueryContext(UUID.randomUUID(), UUID.randomUUID(),
+            Thread.currentThread().getId(), GridH2QueryType.LOCAL);
 
-        for (int i = 1; i < ids.length; i += 2) {
-            p.setObject(1, ids[i]);
+        try {
+            GridH2QueryContext.set(qctx);
 
-            ResultSet rs = p.executeQuery();
+            PreparedStatement p = conn.prepareStatement("select count(*) from t where id = ?");
 
-            assertTrue(rs.next());
+            for (int i = 1; i < ids.length; i += 2) {
+                p.setObject(1, ids[i]);
 
-            assertEquals(1, rs.getInt(1));
-        }
+                ResultSet rs = p.executeQuery();
+
+                assertTrue(rs.next());
 
-        Statement s = conn.createStatement();
+                assertEquals(1, rs.getInt(1));
+            }
+
+            Statement s = conn.createStatement();
 
-        ResultSet rs = s.executeQuery("select count(*) from t");
+            ResultSet rs = s.executeQuery("select count(*) from t");
 
-        assertTrue(rs.next());
+            assertTrue(rs.next());
 
-        assertEquals(ids.length - deleted.get(), rs.getInt(1));
+            assertEquals(ids.length - deleted.get(), rs.getInt(1));
+        }
+        finally {
+            qctx.clearContext(false);
+        }
     }
 
     /**
@@ -525,7 +567,7 @@ public void testRebuildIndexes() throws Exception {
             UUID id = UUID.randomUUID();
 
             GridH2Row row = row(id, System.currentTimeMillis(), rnd.nextBoolean() ? id.toString() :
-                    UUID.randomUUID().toString(), rnd.nextInt(100));
+                UUID.randomUUID().toString(), rnd.nextInt(100));
 
             tbl.doUpdate(row, false);
         }
@@ -569,7 +611,7 @@ private void checkQueryPlan(Connection conn, String sql, String search) throws S
                 String plan = r.getString(1);
 
                 assertTrue("Execution plan for '" + sql + "' query should contain '" + search + "'",
-                        plan.contains(search));
+                    plan.contains(search));
             }
         }
     }
@@ -588,7 +630,7 @@ private Set<Row> checkIndexesConsistent(ArrayList<Index> idxs, @Nullable Set<Row
 
             Iterator<GridH2Row> iter = ((GridH2TreeIndex)idx).rows();
 
-            while(iter.hasNext())
+            while (iter.hasNext())
                 assertTrue(set.add(iter.next()));
 
             //((GridH2SnapTreeSet)((GridH2Index)idx).tree).print();

From 705d9cfca519183fa9e79e0686c1db48c0f5afe2 Mon Sep 17 00:00:00 2001
From: rfqu <rfq@list.ru>
Date: Thu, 1 Jun 2017 19:31:11 +0300
Subject: [PATCH 188/516] ticket fixed: IGN-7062 (TcpDiscoverySpi ignores
 maxMissedClientHeartbeats property)

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 19 +++++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 58 ++++++++++++++++++-
 2 files changed, 72 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 02ba56a884b33..2412624e900fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -366,7 +366,7 @@ else if (state == DISCONNECTED) {
                 else {
                     final GridFutureAdapter<Boolean> finalFut = fut;
 
-                    timer.schedule(new TimerTask() {
+                    TimerTask task = new TimerTask() {
                         @Override public void run() {
                             if (pingFuts.remove(nodeId, finalFut)) {
                                 if (ClientImpl.this.state == DISCONNECTED)
@@ -376,7 +376,13 @@ else if (state == DISCONNECTED) {
                                     finalFut.onDone(false);
                             }
                         }
-                    }, spi.netTimeout);
+                    };
+
+                    try {
+                        timer.schedule(task, spi.netTimeout);
+                    } catch (IllegalStateException e) {
+                        return false;  // timer is cancelled because this client node is dying
+                    }
 
                     sockWriter.sendMessage(new TcpDiscoveryClientPingRequest(getLocalNodeId(), nodeId));
                 }
@@ -803,7 +809,12 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
         U.warn(log, "Simulating client node failure: " + getLocalNodeId());
 
         U.interrupt(sockWriter);
+        U.interrupt(sockReader);
         U.interrupt(msgWorker);
+        try {
+            timer.cancel();
+        } catch (Throwable e) {
+        }
 
         U.join(sockWriter, log);
         U.join(
@@ -870,6 +881,10 @@ private class HeartbeatSender extends TimerTask {
                 msg.client(true);
 
                 sockWriter.sendMessage(msg);
+
+                if (log.isDebugEnabled())
+                    log.debug("*** Send heartbeat message from node:    "+msg.creatorNodeId());
+
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index afd1c2ba7fc66..2769fae3cc5ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6088,9 +6088,11 @@ private void processClientHeartbeatMessage(TcpDiscoveryClientHeartbeatMessage ms
             assert msg.client();
 
             ClientMessageWorker wrk = clientMsgWorkers.get(msg.creatorNodeId());
+            if (wrk != null) {
+                msg.verify(getLocalNodeId());
 
-            if (wrk != null)
-                wrk.metrics(msg.metrics());
+                wrk.addMessage(msg);
+            }
             else if (log.isDebugEnabled())
                 log.debug("Received heartbeat message from unknown client node: " + msg);
         }
@@ -6223,6 +6225,9 @@ private class StatisticsPrinter extends IgniteSpiThread {
     /**
      */
     private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAbstractMessage, byte[]>> {
+        /** minimal period of time which can be used as heartbeat timeout */
+        public static final int minHeartbeaTimeout = 10; // ms
+
         /** Node ID. */
         private final UUID clientNodeId;
 
@@ -6238,15 +6243,25 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
         /** */
         private IgniteProductVersion clientVer;
 
+        /** period of time after which client node which does not send heartbeat messages, is considered dead.
+         * Measured in milliseconds
+         */
+        private long heartBeatTimeOut;
+
+        /** timestamp of the last received heartbeat message, in milliseconds */
+        private long lastHeartBeatTime;
+
         /**
          * @param sock Socket.
          * @param clientNodeId Node ID.
          */
         protected ClientMessageWorker(Socket sock, UUID clientNodeId) throws IOException {
-            super("tcp-disco-client-message-worker", 2000);
+            super("tcp-disco-client-message-worker", Math.max(spi.getHeartbeatFrequency(), minHeartbeaTimeout));
 
             this.sock = sock;
             this.clientNodeId = clientNodeId;
+            this.heartBeatTimeOut = spi.getHeartbeatFrequency() * spi.getMaxMissedClientHeartbeats();
+            this.lastHeartBeatTime=U.currentTimeMillis();
         }
 
         /**
@@ -6295,6 +6310,33 @@ void addMessage(TcpDiscoveryAbstractMessage msg, @Nullable byte[] msgBytes) {
                 log.debug("Message has been added to client queue: " + msg);
         }
 
+
+        /**
+         * Check the last time a heartbeat message received.
+         * In case of timeout, expel client node from the topology
+         */
+        @Override
+        protected void noMessageLoop() {
+            long period = U.currentTimeMillis() - lastHeartBeatTime;
+
+            if (period >= heartBeatTimeOut) {
+                if (log.isInfoEnabled())
+                    log.info("### No heartbeat message from node:" + clientNodeId + "; timeOut=" + heartBeatTimeOut + "; period=" + period);
+
+                TcpDiscoveryAbstractMessage msg = new TcpDiscoveryNodeLeftMessage(clientNodeId);
+
+                msg.senderNodeId(getLocalNodeId());
+
+                msgWorker.addMessage(msg);
+
+                clientMsgWorkers.remove(clientNodeId, this);
+
+                U.interrupt(this);
+
+                U.closeQuiet(sock);
+            }
+        }
+
         /** {@inheritDoc} */
         @Override protected void processMessage(T2<TcpDiscoveryAbstractMessage, byte[]> msgT) {
             boolean success = false;
@@ -6333,6 +6375,16 @@ else if (msgLog.isDebugEnabled())
                             spi.failureDetectionTimeout() : spi.getSocketTimeout());
                     }
                 }
+                else if (msg instanceof TcpDiscoveryClientHeartbeatMessage) {
+                    TcpDiscoveryClientHeartbeatMessage hbmsg = (TcpDiscoveryClientHeartbeatMessage)msg;
+
+                    if (log.isDebugEnabled()) // TODO turn to debug
+                        log.debug("###  Received heartbeat message from node:" + hbmsg.creatorNodeId());
+
+                    this.metrics = hbmsg.metrics();
+
+                    this.lastHeartBeatTime=U.currentTimeMillis();
+                }
                 else {
                     if (msgLog.isDebugEnabled())
                         msgLog.debug("Redirecting message to client [sock=" + sock + ", locNodeId="

From 95d55954de4d60cb1f1f46f19f1aa0b8a9821f16 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 1 Jun 2017 19:56:34 +0300
Subject: [PATCH 189/516] SSL fix

---
 .../ignite/spi/communication/tcp/TcpCommunicationSpi.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index d50bc425e4569..c319848f1b9a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3198,10 +3198,10 @@ else if (log.isDebugEnabled())
                             assert sslHnd != null;
 
                             buf = ByteBuffer.allocate(1000);
+                            buf.order(ByteOrder.nativeOrder());
 
                             ByteBuffer decode = ByteBuffer.allocate(2 * buf.capacity());
-
-                            buf.order(ByteOrder.nativeOrder());
+                            decode.order(ByteOrder.nativeOrder());
 
                             for (int i = 0; i < 9; ) {
                                 int read = ch.read(buf);

From 5f9dc362f82bc6351d01a77418e42c01db9391cf Mon Sep 17 00:00:00 2001
From: rfqu <rfq@list.ru>
Date: Fri, 2 Jun 2017 12:11:40 +0300
Subject: [PATCH 190/516] code style fixed

---
 .../java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java | 1 -
 .../java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java | 5 +++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 2412624e900fe..b83a3809aa26d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -884,7 +884,6 @@ private class HeartbeatSender extends TimerTask {
 
                 if (log.isDebugEnabled())
                     log.debug("*** Send heartbeat message from node:    "+msg.creatorNodeId());
-
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 2769fae3cc5ef..178d22b577fe3 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6243,7 +6243,8 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
         /** */
         private IgniteProductVersion clientVer;
 
-        /** period of time after which client node which does not send heartbeat messages, is considered dead.
+        /**
+         * Period of time after which client node which does not send heartbeat messages, is considered dead.
          * Measured in milliseconds
          */
         private long heartBeatTimeOut;
@@ -6321,7 +6322,7 @@ protected void noMessageLoop() {
 
             if (period >= heartBeatTimeOut) {
                 if (log.isInfoEnabled())
-                    log.info("### No heartbeat message from node:" + clientNodeId + "; timeOut=" + heartBeatTimeOut + "; period=" + period);
+                    log.info("Heartbeat timeout for node:" + clientNodeId + "; timeOut=" + heartBeatTimeOut + "; period=" + period);
 
                 TcpDiscoveryAbstractMessage msg = new TcpDiscoveryNodeLeftMessage(clientNodeId);
 

From 05c639f041ffbbc53678addaf46fc806fb7c168c Mon Sep 17 00:00:00 2001
From: rfqu <rfq@list.ru>
Date: Fri, 2 Jun 2017 12:25:53 +0300
Subject: [PATCH 191/516] coding style fixed

---
 .../java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java     | 1 +
 .../java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java     | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index b83a3809aa26d..9225a922b5dd3 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -814,6 +814,7 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
         try {
             timer.cancel();
         } catch (Throwable e) {
+            // No-op.
         }
 
         U.join(sockWriter, log);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 178d22b577fe3..c7e6d9619ed54 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6383,7 +6383,6 @@ else if (msg instanceof TcpDiscoveryClientHeartbeatMessage) {
                         log.debug("###  Received heartbeat message from node:" + hbmsg.creatorNodeId());
 
                     this.metrics = hbmsg.metrics();
-
                     this.lastHeartBeatTime=U.currentTimeMillis();
                 }
                 else {

From 4d2c9ef1cd50be0a73cd8ac4a0edc809631b23a2 Mon Sep 17 00:00:00 2001
From: rfqu <rfq@list.ru>
Date: Fri, 2 Jun 2017 13:49:31 +0300
Subject: [PATCH 192/516] test added, taken from tiket IGNITE-5103

---
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   3 +-
 .../TcpDiscoveryClientSuspensionSelfTest.java | 116 ++++++++++++++++++
 2 files changed, 117 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index c7e6d9619ed54..318cc23f16101 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6316,8 +6316,7 @@ void addMessage(TcpDiscoveryAbstractMessage msg, @Nullable byte[] msgBytes) {
          * Check the last time a heartbeat message received.
          * In case of timeout, expel client node from the topology
          */
-        @Override
-        protected void noMessageLoop() {
+        @Override protected void noMessageLoop() {
             long period = U.currentTimeMillis() - lastHeartBeatTime;
 
             if (period >= heartBeatTimeOut) {
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
new file mode 100644
index 0000000000000..9dc6e506cc74a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.Timer;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for missed client heartbeats.
+ */
+public class TcpDiscoveryClientSuspensionSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+        disco.setHeartbeatFrequency(200);
+        disco.setMaxMissedClientHeartbeats(10);
+
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneServer() throws Exception {
+        doTestClientSuspension(1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTwoServers() throws Exception {
+        doTestClientSuspension(2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testThreeServers() throws Exception {
+        doTestClientSuspension(3);
+    }
+
+    /**
+     * @param serverCnt Servers count.
+     * @throws Exception If failed.
+     */
+    private void doTestClientSuspension(int serverCnt) throws Exception {
+        startGrids(serverCnt);
+
+        Ignition.setClientMode(true);
+
+        Ignite client = startGrid("client");
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(1, grid(i).cluster().forClients().nodes().size());
+
+        Thread.sleep(3000);
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(1, grid(i).cluster().forClients().nodes().size());
+
+        suspendClientHeartbeats(client);
+
+        Thread.sleep(3000);
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(0, grid(i).cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @param client Client.
+     */
+    private void suspendClientHeartbeats(Ignite client) {
+        assert client.cluster().localNode().isClient();
+
+        ClientImpl impl = U.field(client.configuration().getDiscoverySpi(), "impl");
+
+        impl.simulateNodeFailure();
+
+        //Timer timer = U.field(impl, "timer"); timer.cancel(); -- client node successfully reconnects
+    }
+}

From 744a81ba937ba83ecdefa7c71f198d92d21527bb Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Wed, 31 May 2017 15:27:33 +0300
Subject: [PATCH 193/516] IGNITE-5232 [BACKPORT]
 GridDhtPartitionDemander.requestPartitions invokes sendMessages consequently,
 which lead to significant increase of node start time on large clusters with
 ssl

---
 .../preloader/GridDhtPartitionDemander.java   | 131 ++++++++++--------
 1 file changed, 71 insertions(+), 60 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index a6808c73577e1..daae1e2aa144a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -331,41 +331,21 @@ Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
 
             return new Runnable() {
                 @Override public void run() {
-                    try {
-                        if (next != null)
-                            fut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
-                                @Override public void apply(IgniteInternalFuture<Boolean> f) {
-                                    try {
-                                        if (f.get()) // Not cancelled.
-                                            next.run(); // Starts next cache rebalancing (according to the order).
-                                    }
-                                    catch (IgniteCheckedException ignored) {
-                                        if (log.isDebugEnabled())
-                                            log.debug(ignored.getMessage());
-                                    }
+                    if (next != null)
+                        fut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
+                            @Override public void apply(IgniteInternalFuture<Boolean> f) {
+                                try {
+                                    if (f.get()) // Not cancelled.
+                                        next.run(); // Starts next cache rebalancing (according to the order).
                                 }
-                            });
-
-                        requestPartitions(fut, assigns);
-                    }
-                    catch (IgniteCheckedException e) {
-                        ClusterTopologyCheckedException cause = e.getCause(ClusterTopologyCheckedException.class);
-
-                        if (cause != null)
-                            log.warning("Failed to send initial demand request to node. " + e.getMessage());
-                        else
-                            log.error("Failed to send initial demand request to node.", e);
-
-                        fut.cancel();
-                    }
-                    catch (Throwable th) {
-                        log.error("Runtime error caught during initial demand request sending.", th);
-
-                        fut.cancel();
+                                catch (IgniteCheckedException ignored) {
+                                    if (log.isDebugEnabled())
+                                        log.debug(ignored.getMessage());
+                                }
+                            }
+                        });
 
-                        if (th instanceof Error)
-                            throw th;
-                    }
+                    requestPartitions(fut, assigns);
                 }
             };
         }
@@ -404,9 +384,8 @@ else if (delay > 0) {
      * @return Partitions were requested.
      */
     private void requestPartitions(
-        RebalanceFuture fut,
-        GridDhtPreloaderAssignments assigns
-    ) throws IgniteCheckedException {
+        final RebalanceFuture fut,
+        GridDhtPreloaderAssignments assigns){
         if (topologyChanged(fut)) {
             fut.cancel();
 
@@ -438,7 +417,7 @@ private void requestPartitions(
 
                 int lsnrCnt = cctx.gridConfig().getRebalanceThreadPoolSize();
 
-                List<Set<Integer>> sParts = new ArrayList<>(lsnrCnt);
+            final List<Set<Integer>> sParts = new ArrayList<>(lsnrCnt);
 
                 for (int cnt = 0; cnt < lsnrCnt; cnt++)
                     sParts.add(new HashSet<Integer>());
@@ -453,42 +432,74 @@ private void requestPartitions(
                 for (cnt = 0; cnt < lsnrCnt; cnt++) {
                     if (!sParts.get(cnt).isEmpty()) {
                         // Create copy.
-                        GridDhtPartitionDemandMessage initD = new GridDhtPartitionDemandMessage(d, sParts.get(cnt));
+                        final GridDhtPartitionDemandMessage initD = new GridDhtPartitionDemandMessage(d, sParts.get(cnt));
 
                         initD.topic(rebalanceTopics.get(cnt));
                         initD.updateSequence(fut.updateSeq);
                         initD.timeout(cctx.config().getRebalanceTimeout());
 
-                        synchronized (fut) {
-                            if (!fut.isDone()) {
-                                // Future can be already cancelled at this moment and all failovers happened.
-                                // New requests will not be covered by failovers.
-                                cctx.io().sendOrderedMessage(node,
-                                    rebalanceTopics.get(cnt), initD, cctx.ioPolicy(), initD.timeout());
-                            }
-                        }
+                        final int finalCnt = cnt;
 
-                        if (log.isDebugEnabled())
-                            log.debug("Requested rebalancing [from node=" + node.id() + ", listener index=" +
-                                cnt + ", partitions count=" + sParts.get(cnt).size() +
-                                " (" + partitionsList(sParts.get(cnt)) + ")]");
+                        cctx.kernalContext().closure().runLocalSafe(new Runnable() {
+                            @Override public void run() {
+                                try {
+                                    if (!fut.isDone()) {
+                                        cctx.io().sendOrderedMessage(node,
+                                            rebalanceTopics.get(finalCnt), initD, cctx.ioPolicy(), initD.timeout());
+
+                                        // Cleanup required in case partitions demanded in parallel with cancellation.
+                                        synchronized (fut) {
+                                            if (fut.isDone())
+                                                fut.cleanupRemoteContexts(node.id());
+                                        }
+
+                                        if (log.isDebugEnabled())
+                                            log.debug("Requested rebalancing [from node=" + node.id() + ", listener index=" +
+                                                finalCnt + ", partitions count=" + sParts.get(finalCnt).size() +
+                                                " (" + partitionsList(sParts.get(finalCnt)) + ")]");
+                                    }
+                                }
+                                catch (IgniteCheckedException e) {
+                                    ClusterTopologyCheckedException cause = e.getCause(ClusterTopologyCheckedException.class);
+
+                                    if (cause != null)
+                                        log.warning("Failed to send initial demand request to node. " + e.getMessage());
+                                    else
+                                        log.error("Failed to send initial demand request to node.", e);
+
+                                    fut.cancel();
+                                }
+                                catch (Throwable th) {
+                                    log.error("Runtime error caught during initial demand request sending.", th);
+
+                                    fut.cancel();
+                                }
+                            }
+                        }, /*system pool*/true);
                     }
                 }
             }
             else {
-                U.log(log, "Starting rebalancing (old api) [cache=" + cctx.name() +
-                    ", mode=" + cfg.getRebalanceMode() +
-                    ", fromNode=" + node.id() +
-                    ", partitionsCount=" + parts.size() +
-                    ", topology=" + fut.topologyVersion() +
-                    ", updateSeq=" + fut.updateSeq + "]");
+                try {
+                    U.log(log, "Starting rebalancing (old api) [cache=" + cctx.name() +
+                        ", mode=" + cfg.getRebalanceMode() +
+                        ", fromNode=" + node.id() +
+                        ", partitionsCount=" + parts.size() +
+                        ", topology=" + fut.topologyVersion() +
+                        ", updateSeq=" + fut.updateSeq + "]");
 
-                d.timeout(cctx.config().getRebalanceTimeout());
-                d.workerId(0);//old api support.
+                    d.timeout(cctx.config().getRebalanceTimeout());
+                    d.workerId(0);//old api support.
+
+                    worker = new DemandWorker(dmIdx.incrementAndGet(), fut);
 
-                worker = new DemandWorker(dmIdx.incrementAndGet(), fut);
+                    worker.run(node, d);
+                }
+                catch (Throwable th) {
+                    log.error("Runtime error caught during initial demand request sending.", th);
 
-                worker.run(node, d);
+                    fut.cancel();
+                }
             }
         }
     }

From 8220fb121eec86c18a711816f3db478b1d31a4e6 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 24 May 2017 19:55:09 +0300
Subject: [PATCH 194/516] ignite-5283 Fix of transaction recovery on backup
 when primary node failed

---
 .../GridDistributedLockRequest.java           |  20 ++
 .../distributed/dht/GridDhtLockFuture.java    |   1 +
 .../distributed/dht/GridDhtLockRequest.java   |   4 +
 .../dht/GridDhtTransactionalCacheAdapter.java |   3 +-
 .../dht/GridDhtTxPrepareFuture.java           |   2 +
 .../dht/GridDhtTxPrepareRequest.java          |  25 +-
 .../distributed/dht/GridDhtTxRemote.java      |  24 +-
 .../cache/transactions/IgniteTxHandler.java   |   3 +-
 .../IgniteTxCachePrimarySyncTest.java         |  35 ++-
 .../IgniteCachePutRetryAbstractSelfTest.java  |  48 ++++
 .../dht/TxRecoveryStoreEnabledTest.java       | 227 ++++++++++++++++++
 .../IgniteCacheTxRecoverySelfTestSuite.java   |   3 +
 12 files changed, 375 insertions(+), 20 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
index 9639a9a7288ee..3142c8ba836ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
@@ -48,6 +48,9 @@ public class GridDistributedLockRequest extends GridDistributedBaseMessage {
     /** */
     private static final long serialVersionUID = 0L;
 
+    /** */
+    private static final int STORE_USED_FLAG_MASK = 0x04;
+
     /** Sender node ID. */
     private UUID nodeId;
 
@@ -262,6 +265,23 @@ public boolean keepBinary() {
         return (flags & KEEP_BINARY_FLAG_MASK) != 0;
     }
 
+    /**
+     * @return Flag indicating whether transaction use cache store.
+     */
+    public boolean storeUsed() {
+        return (flags & STORE_USED_FLAG_MASK) != 0;
+    }
+
+    /**
+     * @param storeUsed Store used value.
+     */
+    public void storeUsed(boolean storeUsed) {
+        if (storeUsed)
+            flags = (byte)(flags | STORE_USED_FLAG_MASK);
+        else
+            flags &= ~STORE_USED_FLAG_MASK;
+    }
+
     /**
      * @return Transaction isolation or <tt>null</tt> if not in transaction.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
index 686a4c6bed872..ddeb1ad1da507 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
@@ -873,6 +873,7 @@ private void map(Iterable<GridDhtCacheEntry> entries) {
                         inTx() ? tx.taskNameHash() : 0,
                         read ? accessTtl : -1L,
                         skipStore,
+                        cctx.store().configured(),
                         keepBinary,
                         cctx.deploymentEnabled());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
index 95c6dfc06ce44..289ad93d884fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
@@ -120,6 +120,7 @@ public GridDhtLockRequest() {
      * @param taskNameHash Task name hash code.
      * @param accessTtl TTL for read operation.
      * @param skipStore Skip store flag.
+     * @param storeUsed Cache store configured flag.
      * @param keepBinary Keep binary flag.
      * @param addDepInfo Deployment info flag.
      */
@@ -144,6 +145,7 @@ public GridDhtLockRequest(
         int taskNameHash,
         long accessTtl,
         boolean skipStore,
+        boolean storeUsed,
         boolean keepBinary,
         boolean addDepInfo
     ) {
@@ -166,6 +168,8 @@ public GridDhtLockRequest(
 
         this.topVer = topVer;
 
+        storeUsed(storeUsed);
+
         nearKeys = nearCnt == 0 ? Collections.<KeyCacheObject>emptyList() : new ArrayList<KeyCacheObject>(nearCnt);
         invalidateEntries = new BitSet(dhtCnt == 0 ? nearCnt : dhtCnt);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index a9e3bc496721c..13a88e9720ed7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -245,7 +245,8 @@ protected GridDhtTransactionalCacheAdapter(GridCacheContext<K, V> ctx, GridCache
                                     req.timeout(),
                                     req.txSize(),
                                     req.subjectId(),
-                                    req.taskNameHash());
+                                    req.taskNameHash(),
+                                    !req.skipStore() && req.storeUsed());
 
                                 tx = ctx.tm().onCreated(null, tx);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index a759194453a78..4283ccb878a47 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -1247,6 +1247,7 @@ private void prepare0() {
                         tx.subjectId(),
                         tx.taskNameHash(),
                         tx.activeCachesDeploymentEnabled(),
+                        tx.storeUsed(),
                         retVal);
 
                     int idx = 0;
@@ -1359,6 +1360,7 @@ private void prepare0() {
                             tx.subjectId(),
                             tx.taskNameHash(),
                             tx.activeCachesDeploymentEnabled(),
+                            tx.storeUsed(),
                             retVal);
 
                         for (IgniteTxEntry entry : nearMapping.entries()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
index a8f20876abd29..9fa20d8b75b65 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
@@ -55,6 +55,9 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
     /** */
     public static final int NEED_RETURN_VALUE_FLAG_MASK = 0x01;
 
+    /** */
+    public static final int STORE_USED_FLAG_MASK = 0x02;
+
     /** Max order. */
     private UUID nearNodeId;
 
@@ -124,8 +127,9 @@ public GridDhtTxPrepareRequest() {
      * @param txNodes Transaction nodes mapping.
      * @param nearXidVer Near transaction ID.
      * @param last {@code True} if this is last prepare request for node.
-     * @param retVal Need return value flag.
      * @param addDepInfo Deployment info flag.
+     * @param storeUsed Cache store used flag.
+     * @param retVal Need return value flag.
      */
     public GridDhtTxPrepareRequest(
         IgniteUuid futId,
@@ -142,6 +146,7 @@ public GridDhtTxPrepareRequest(
         UUID subjId,
         int taskNameHash,
         boolean addDepInfo,
+        boolean storeUsed,
         boolean retVal) {
         super(tx, timeout, null, dhtWrites, txNodes, onePhaseCommit, addDepInfo);
 
@@ -157,6 +162,7 @@ public GridDhtTxPrepareRequest(
         this.subjId = subjId;
         this.taskNameHash = taskNameHash;
 
+        storeUsed(storeUsed);
         needReturnValue(retVal);
 
         invalidateNearEntries = new BitSet(dhtWrites == null ? 0 : dhtWrites.size());
@@ -181,6 +187,23 @@ public void needReturnValue(boolean retVal) {
             flags &= ~NEED_RETURN_VALUE_FLAG_MASK;
     }
 
+    /**
+     * @return Flag indicating whether transaction use cache store.
+     */
+    public boolean storeUsed() {
+        return (flags & STORE_USED_FLAG_MASK) != 0;
+    }
+
+    /**
+     * @param storeUsed Store used value.
+     */
+    public void storeUsed(boolean storeUsed) {
+        if (storeUsed)
+            flags = (byte)(flags | STORE_USED_FLAG_MASK);
+        else
+            flags &= ~STORE_USED_FLAG_MASK;
+    }
+
     /**
      * @return {@code True} if this is last prepare request for node.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
index 6ad20c7566553..8942ef9d9178c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
@@ -64,6 +64,9 @@ public class GridDhtTxRemote extends GridDistributedTxRemoteAdapter {
     /** Near transaction ID. */
     private GridCacheVersion nearXidVer;
 
+    /** Store used. */
+    private boolean storeUsed;
+
     /**
      * Empty constructor required for {@link Externalizable}.
      */
@@ -74,6 +77,7 @@ public GridDhtTxRemote() {
     /**
      * This constructor is meant for optimistic transactions.
      *
+     * @param ctx Cache context.
      * @param nearNodeId Near node ID.
      * @param rmtFutId Remote future ID.
      * @param nodeId Node ID.
@@ -85,10 +89,10 @@ public GridDhtTxRemote() {
      * @param isolation Transaction isolation.
      * @param invalidate Invalidate flag.
      * @param timeout Timeout.
-     * @param ctx Cache context.
      * @param txSize Expected transaction size.
      * @param nearXidVer Near transaction ID.
      * @param txNodes Transaction nodes mapping.
+     * @param storeUsed Cache store used flag.
      */
     public GridDhtTxRemote(
         GridCacheSharedContext ctx,
@@ -109,8 +113,8 @@ public GridDhtTxRemote(
         Map<UUID, Collection<UUID>> txNodes,
         @Nullable UUID subjId,
         int taskNameHash,
-        boolean single
-    ) {
+        boolean single,
+        boolean storeUsed) {
         super(
             ctx,
             nodeId,
@@ -134,6 +138,7 @@ public GridDhtTxRemote(
         this.rmtFutId = rmtFutId;
         this.nearXidVer = nearXidVer;
         this.txNodes = txNodes;
+        this.storeUsed = storeUsed;
 
         txState = single ? new IgniteTxRemoteSingleStateImpl() :
             new IgniteTxRemoteStateImpl(
@@ -148,6 +153,7 @@ public GridDhtTxRemote(
     /**
      * This constructor is meant for pessimistic transactions.
      *
+     * @param ctx Cache context.
      * @param nearNodeId Near node ID.
      * @param rmtFutId Remote future ID.
      * @param nodeId Node ID.
@@ -160,8 +166,8 @@ public GridDhtTxRemote(
      * @param isolation Transaction isolation.
      * @param invalidate Invalidate flag.
      * @param timeout Timeout.
-     * @param ctx Cache context.
      * @param txSize Expected transaction size.
+     * @param storeUsed Cache store used flag.
      */
     public GridDhtTxRemote(
         GridCacheSharedContext ctx,
@@ -180,8 +186,8 @@ public GridDhtTxRemote(
         long timeout,
         int txSize,
         @Nullable UUID subjId,
-        int taskNameHash
-    ) {
+        int taskNameHash,
+        boolean storeUsed) {
         super(
             ctx,
             nodeId,
@@ -204,6 +210,7 @@ public GridDhtTxRemote(
         this.nearXidVer = nearXidVer;
         this.nearNodeId = nearNodeId;
         this.rmtFutId = rmtFutId;
+        this.storeUsed = storeUsed;
 
         txState = new IgniteTxRemoteStateImpl(
             Collections.<IgniteTxKey, IgniteTxEntry>emptyMap(),
@@ -226,6 +233,11 @@ public void transactionNodes(Map<UUID, Collection<UUID>> txNodes) {
         return true;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean storeUsed() {
+        return storeUsed;
+    }
+
     /** {@inheritDoc} */
     @Override public UUID eventNodeId() {
         return nearNodeId();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index d56415652bc73..71bf08cfdfdbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -1417,7 +1417,8 @@ protected void sendReply(UUID nodeId, GridDhtTxFinishRequest req, boolean commit
                     req.transactionNodes(),
                     req.subjectId(),
                     req.taskNameHash(),
-                    single);
+                    single,
+                    req.storeUsed());
 
                 tx.writeVersion(req.writeVersion());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java
index 3bc22ef8c05c7..020ec5c8646d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
 import javax.cache.Cache;
 import javax.cache.configuration.Factory;
 import javax.cache.integration.CacheLoaderException;
@@ -62,6 +63,7 @@
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
+import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
@@ -1096,19 +1098,30 @@ private static class TestStoreFactory implements Factory<CacheStore<Object, Obje
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
         @Override public CacheStore<Object, Object> create() {
-            return new CacheStoreAdapter() {
-                @Override public Object load(Object key) throws CacheLoaderException {
-                    return null;
-                }
+            return new TestCacheStore();
+        }
+    }
 
-                @Override public void write(Cache.Entry entry) throws CacheWriterException {
-                    // No-op.
-                }
+    /**
+     *
+     */
+    private static class TestCacheStore extends CacheStoreAdapter<Object, Object> {
+        /** Store map. */
+        private static final Map STORE_MAP = new ConcurrentHashMap();
 
-                @Override public void delete(Object key) throws CacheWriterException {
-                    // No-op.
-                }
-            };
+        /** {@inheritDoc} */
+        @Override public Object load(Object key) throws CacheLoaderException {
+            return STORE_MAP.get(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<?, ?> entry) throws CacheWriterException {
+            STORE_MAP.put(entry.getKey(), entry.getValue());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            STORE_MAP.remove(key);
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
index c3d194b0c4f27..fc9017927d3bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
@@ -24,6 +24,7 @@
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.cache.Cache;
 import javax.cache.configuration.Factory;
@@ -56,6 +57,7 @@
 import org.apache.ignite.spi.swapspace.inmemory.GridTestSwapSpaceSpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -650,4 +652,50 @@ private static class TestStoreFactory implements Factory<CacheStore> {
             };
         }
     }
+
+
+/*
+    private static class TestStoreFactory implements Factory<CacheStore> {
+        */
+/** {@inheritDoc} *//*
+
+        @Override public CacheStore create() {
+            return new TestCacheStore();
+        }
+    }
+
+    */
+/**
+     *
+     *//*
+
+    private static class TestCacheStore extends CacheStoreAdapter {
+        */
+/** Store map. *//*
+
+        private static Map STORE_MAP = new ConcurrentHashMap();
+
+        */
+/** {@inheritDoc} *//*
+
+        @Override public Object load(Object key) throws CacheLoaderException {
+            return STORE_MAP.get(key);
+        }
+
+        */
+/** {@inheritDoc} *//*
+
+        @Override public void write(Cache.Entry entry) throws CacheWriterException {
+            STORE_MAP.put(entry.getKey(), entry.getValue());
+        }
+
+        */
+/** {@inheritDoc} *//*
+
+        @Override public void delete(Object key) throws CacheWriterException {
+            STORE_MAP.remove(key);
+        }
+    }
+*/
+
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
new file mode 100644
index 0000000000000..73f0268c42ef3
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.concurrent.CountDownLatch;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+
+/**
+ *
+ */
+public class TxRecoveryStoreEnabledTest extends GridCommonAbstractTest {
+    /** Nodes count. */
+    private static final int NODES_CNT = 2;
+
+    /** Latch. */
+    private static CountDownLatch latch;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCommunicationSpi(new TestCommunicationSpi());
+        cfg.setDiscoverySpi(new TestDiscoverySpi());
+
+        CacheConfiguration ccfg = defaultCacheConfiguration();
+
+        ccfg.setNearConfiguration(null);
+        ccfg.setCacheMode(CacheMode.PARTITIONED);
+        ccfg.setBackups(1);
+        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+        ccfg.setCacheStoreFactory(new TestCacheStoreFactory());
+        ccfg.setReadThrough(true);
+        ccfg.setWriteThrough(true);
+        ccfg.setWriteBehindEnabled(false);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        latch = new CountDownLatch(1);
+
+        startGrids(NODES_CNT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOptimistic() throws Exception {
+        checkTxRecovery(OPTIMISTIC);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPessimistic() throws Exception {
+        checkTxRecovery(PESSIMISTIC);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkTxRecovery(TransactionConcurrency concurrency) throws Exception {
+        final Ignite node0 = ignite(0);
+        Ignite node1 = ignite(1);
+
+        IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    latch.await();
+
+                    IgniteConfiguration cfg = node0.configuration();
+
+                    ((TestCommunicationSpi)cfg.getCommunicationSpi()).block();
+                    ((TestDiscoverySpi)cfg.getDiscoverySpi()).simulateNodeFailure();
+                }
+                catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                }
+            }
+        }, 1);
+
+        IgniteCache<Object, Object> cache0 = node0.cache(null);
+
+        Integer key = primaryKey(cache0);
+
+        try (Transaction tx = node0.transactions().txStart(concurrency, READ_COMMITTED)) {
+            cache0.put(key, key);
+
+            tx.commit();
+        }
+        catch (Exception e) {
+            // No-op.
+        }
+
+        fut.get();
+
+        IgniteCache<Object, Object> cache1 = node1.cache(null);
+
+        assertNull(cache1.get(key));
+    }
+
+    /**
+     *
+     */
+    private static class TestCacheStoreFactory implements Factory<CacheStore> {
+        /** {@inheritDoc} */
+        @Override public CacheStore create() {
+            return new TestCacheStore();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestCacheStore extends CacheStoreAdapter<Integer, Integer> {
+        /** {@inheritDoc} */
+        @Override public void sessionEnd(boolean commit) {
+            if (latch.getCount() > 0) { // Need wait only on primary node.
+                latch.countDown();
+
+                try {
+                    U.sleep(3000);
+                }
+                catch (IgniteInterruptedCheckedException e) {
+                    Thread.currentThread().interrupt();
+                }
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer load(Integer key) throws CacheLoaderException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Integer, ? extends Integer> entry) throws CacheWriterException {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            // no-op.
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void simulateNodeFailure() {
+            super.simulateNodeFailure();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** Block. */
+        private volatile boolean block;
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+            if (!block)
+                super.sendMessage(node, msg);
+        }
+
+        /**
+         *
+         */
+        private void block() {
+            block = true;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
index 7363c7cc1f2a8..c7c8db67fe31e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedNearDisabledPrimaryNodeFailureRecoveryTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedPrimaryNodeFailureRecoveryTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedTwoBackupsPrimaryNodeFailureRecoveryTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.TxRecoveryStoreEnabledTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearTxPessimisticOriginatingNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxOriginatingNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxPessimisticOriginatingNodeFailureSelfTest;
@@ -54,6 +55,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheNearTxPessimisticOriginatingNodeFailureSelfTest.class);
         suite.addTestSuite(GridCacheReplicatedTxPessimisticOriginatingNodeFailureSelfTest.class);
 
+        suite.addTestSuite(TxRecoveryStoreEnabledTest.class);
+
         return suite;
     }
 }
\ No newline at end of file

From 374cba8a2b0d4438b46258a4ea89e43ab1e7989c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 6 Jun 2017 16:17:01 +0300
Subject: [PATCH 195/516] IGNITE-5259 Minor serialization fix

---
 .../discovery/GridDiscoveryManager.java       |  6 ++---
 .../processors/security/SecurityUtils.java    | 23 +++++++++++++++++--
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 17 +++++++++++++-
 3 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index dfd4c87ec9cae..c27f2a35b92c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -138,8 +138,8 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME;
 import static org.apache.ignite.internal.IgniteVersionUtils.VER;
-import static org.apache.ignite.internal.processors.security.SecurityUtils.SERVICE_PERMISSIONS_SINCE;
 import static org.apache.ignite.internal.processors.security.SecurityUtils.isSecurityCompatibilityMode;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.isServiceSecuritySupported;
 import static org.apache.ignite.plugin.segmentation.SegmentationPolicy.NOOP;
 
 /**
@@ -1163,7 +1163,7 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                     ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
             }
 
-            if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) >= 0
+            if (isServiceSecuritySupported(n.version())
                 && ctx.security().enabled() // Matters only if security enabled.
                ) {
                 Boolean rmtSecurityCompatibilityEnabled = n.attribute(ATTR_SECURITY_COMPATIBILITY_MODE);
@@ -1181,7 +1181,7 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                 }
             }
 
-            if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) < 0
+            if (!isServiceSecuritySupported(n.version())
                 && ctx.security().enabled() // Matters only if security enabled.
                 && (locSecurityCompatibilityEnabled == null || !locSecurityCompatibilityEnabled)) {
                 throw new IgniteCheckedException("Remote node does not support service security permissions. " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
index 1016335888b3d..c84e53f729b66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
@@ -29,8 +29,12 @@
  * Security utilities.
  */
 public class SecurityUtils {
-    /** Version since service security supported. */
-    public static final IgniteProductVersion SERVICE_PERMISSIONS_SINCE = IgniteProductVersion.fromString("1.7.11");
+    /** Versions since service security supported. */
+    private static final IgniteProductVersion[] SERVICE_PERMISSIONS_SINCE = {
+        IgniteProductVersion.fromString("1.7.11"),
+        IgniteProductVersion.fromString("1.8.7"),
+        IgniteProductVersion.fromString("1.9.3")
+    };
 
     /** Default serialization version. */
     private final static int DFLT_SERIALIZE_VERSION = isSecurityCompatibilityMode() ? 1 : 2;
@@ -89,4 +93,19 @@ public static Map<String, Collection<SecurityPermission>> compatibleServicePermi
 
         return srvcPerms;
     }
+
+    /**
+     * Checks whether provided release supports service security permissions.
+     *
+     * @param ver Version to ckeck.
+     * @return {@code True} if passed release supports service security permissions.
+     */
+    public static boolean isServiceSecuritySupported(IgniteProductVersion ver) {
+        for (IgniteProductVersion v : SERVICE_PERMISSIONS_SINCE) {
+            if (v.major() == ver.major() && v.minor() == ver.minor())
+                return ver.compareToIgnoreTimestamp(v) >= 0;
+        }
+
+        return ver.compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE[SERVICE_PERMISSIONS_SINCE.length]) >= 0;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index a5a4e88990c4a..c73fa3987ece3 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.processors.security.SecurityUtils;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
@@ -516,7 +517,16 @@ else if (state == DISCONNECTED) {
 
                 InetSocketAddress addr = it.next();
 
-                T3<SocketStream, Integer, Boolean> sockAndRes = sendJoinRequest(recon, addr);
+                T3<SocketStream, Integer, Boolean> sockAndRes;
+
+                SecurityUtils.serializeVersion(1);
+
+                try {
+                    sockAndRes = sendJoinRequest(recon, addr);
+                }
+                finally {
+                    SecurityUtils.restoreDefaultSerializeVersion();
+                }
 
                 if (sockAndRes == null) {
                     it.remove();
@@ -975,6 +985,8 @@ private void forceStopRead() throws InterruptedException {
                     while (!isInterrupted()) {
                         TcpDiscoveryAbstractMessage msg;
 
+                        SecurityUtils.serializeVersion(1);
+
                         try {
                             msg = U.unmarshal(spi.marshaller(), in, U.resolveClassLoader(spi.ignite().configuration()));
                         }
@@ -1000,6 +1012,9 @@ private void forceStopRead() throws InterruptedException {
 
                             continue;
                         }
+                        finally {
+                            SecurityUtils.restoreDefaultSerializeVersion();
+                        }
 
                         msg.senderNodeId(rmtNodeId);
 

From 5cb580ad7043f27e4a0396aea1f877c21d49078e Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 6 Jun 2017 16:17:01 +0300
Subject: [PATCH 196/516] IGNITE-5259 Minor serialization fix

(cherry picked from commit 374cba8)
---
 .../discovery/GridDiscoveryManager.java       |  6 ++---
 .../processors/security/SecurityUtils.java    | 22 +++++++++++++++++--
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 17 +++++++++++++-
 3 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index fb72989eb81ba..71b7217bd6afc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -138,8 +138,8 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME;
 import static org.apache.ignite.internal.IgniteVersionUtils.VER;
-import static org.apache.ignite.internal.processors.security.SecurityUtils.SERVICE_PERMISSIONS_SINCE;
 import static org.apache.ignite.internal.processors.security.SecurityUtils.isSecurityCompatibilityMode;
+import static org.apache.ignite.internal.processors.security.SecurityUtils.isServiceSecuritySupported;
 import static org.apache.ignite.plugin.segmentation.SegmentationPolicy.NOOP;
 
 /**
@@ -1163,7 +1163,7 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                     ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
             }
 
-            if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) >= 0
+            if (isServiceSecuritySupported(n.version())
                 && ctx.security().enabled() // Matters only if security enabled.
                ) {
                 Boolean rmtSecurityCompatibilityEnabled = n.attribute(ATTR_SECURITY_COMPATIBILITY_MODE);
@@ -1181,7 +1181,7 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                 }
             }
 
-            if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) < 0
+            if (!isServiceSecuritySupported(n.version())
                 && ctx.security().enabled() // Matters only if security enabled.
                 && (locSecurityCompatibilityEnabled == null || !locSecurityCompatibilityEnabled)) {
                 throw new IgniteCheckedException("Remote node does not support service security permissions. " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
index 1016335888b3d..306d8de58d87f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
@@ -29,8 +29,11 @@
  * Security utilities.
  */
 public class SecurityUtils {
-    /** Version since service security supported. */
-    public static final IgniteProductVersion SERVICE_PERMISSIONS_SINCE = IgniteProductVersion.fromString("1.7.11");
+    /** Versions since service security supported. */
+    private static final IgniteProductVersion[] SERVICE_PERMISSIONS_SINCE = {
+        IgniteProductVersion.fromString("1.7.11"),
+        IgniteProductVersion.fromString("1.8.7")
+    };
 
     /** Default serialization version. */
     private final static int DFLT_SERIALIZE_VERSION = isSecurityCompatibilityMode() ? 1 : 2;
@@ -89,4 +92,19 @@ public static Map<String, Collection<SecurityPermission>> compatibleServicePermi
 
         return srvcPerms;
     }
+
+    /**
+     * Checks whether provided release supports service security permissions.
+     *
+     * @param ver Version to ckeck.
+     * @return {@code True} if passed release supports service security permissions.
+     */
+    public static boolean isServiceSecuritySupported(IgniteProductVersion ver) {
+        for (IgniteProductVersion v : SERVICE_PERMISSIONS_SINCE) {
+            if (v.major() == ver.major() && v.minor() == ver.minor())
+                return ver.compareToIgnoreTimestamp(v) >= 0;
+        }
+
+        return ver.compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE[SERVICE_PERMISSIONS_SINCE.length]) >= 0;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 9225a922b5dd3..e7bc81e5bc55f 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.processors.security.SecurityUtils;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
@@ -522,7 +523,16 @@ else if (state == DISCONNECTED) {
 
                 InetSocketAddress addr = it.next();
 
-                T3<SocketStream, Integer, Boolean> sockAndRes = sendJoinRequest(recon, addr);
+                T3<SocketStream, Integer, Boolean> sockAndRes;
+
+                SecurityUtils.serializeVersion(1);
+
+                try {
+                    sockAndRes = sendJoinRequest(recon, addr);
+                }
+                finally {
+                    SecurityUtils.restoreDefaultSerializeVersion();
+                }
 
                 if (sockAndRes == null) {
                     it.remove();
@@ -984,6 +994,8 @@ private void forceStopRead() throws InterruptedException {
                     while (!isInterrupted()) {
                         TcpDiscoveryAbstractMessage msg;
 
+                        SecurityUtils.serializeVersion(1);
+
                         try {
                             msg = U.unmarshal(spi.marshaller(), in, U.resolveClassLoader(spi.ignite().configuration()));
                         }
@@ -1009,6 +1021,9 @@ private void forceStopRead() throws InterruptedException {
 
                             continue;
                         }
+                        finally {
+                            SecurityUtils.restoreDefaultSerializeVersion();
+                        }
 
                         msg.senderNodeId(rmtNodeId);
 

From f03252f9b2c6f0e777f307fd85cc8bd20ab21423 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 6 Jun 2017 16:17:01 +0300
Subject: [PATCH 197/516] IGNITE-5259 Minor serialization fix

(cherry picked from commit 374cba8)
---
 .../ignite/spi/discovery/tcp/ClientImpl.java    | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 02ba56a884b33..b075dc1fb5289 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.processors.security.SecurityUtils;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
@@ -516,7 +517,16 @@ else if (state == DISCONNECTED) {
 
                 InetSocketAddress addr = it.next();
 
-                T3<SocketStream, Integer, Boolean> sockAndRes = sendJoinRequest(recon, addr);
+                T3<SocketStream, Integer, Boolean> sockAndRes;
+
+                SecurityUtils.serializeVersion(1);
+
+                try {
+                    sockAndRes = sendJoinRequest(recon, addr);
+                }
+                finally {
+                    SecurityUtils.restoreDefaultSerializeVersion();
+                }
 
                 if (sockAndRes == null) {
                     it.remove();
@@ -969,6 +979,8 @@ private void forceStopRead() throws InterruptedException {
                     while (!isInterrupted()) {
                         TcpDiscoveryAbstractMessage msg;
 
+                        SecurityUtils.serializeVersion(1);
+
                         try {
                             msg = U.unmarshal(spi.marshaller(), in, U.resolveClassLoader(spi.ignite().configuration()));
                         }
@@ -994,6 +1006,9 @@ private void forceStopRead() throws InterruptedException {
 
                             continue;
                         }
+                        finally {
+                            SecurityUtils.restoreDefaultSerializeVersion();
+                        }
 
                         msg.senderNodeId(rmtNodeId);
 

From d2bf9619aaf867f251bc193d913dd4cc174a33a3 Mon Sep 17 00:00:00 2001
From: Ivan Veselovskiy <iveselovskiy@gridgain.com>
Date: Tue, 6 Jun 2017 16:56:09 +0300
Subject: [PATCH 198/516] IGNITE-5410: Fixed assertion in HadoopDataOutStream.
 This closes #2084.

---
 .../processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
index acc9be62cf677..655346eaae93a 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
@@ -92,7 +92,7 @@ public void pointer(long ptr) {
      * @return Old position pointer or {@code 0} if move goes beyond the end of the buffer.
      */
     public long move(long size) {
-        assert size > 0 : size;
+        assert size >= 0 : size;
 
         long oldPos = posPtr;
         long newPos = oldPos + size;

From 77ff30cc08dae653c0b914167088e9e90cdadd32 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 6 Jun 2017 17:12:27 +0300
Subject: [PATCH 199/516] IGNITE-5259 Minor serialization fix

---
 .../ignite/internal/processors/security/SecurityUtils.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
index c84e53f729b66..3d8b8ccad2c92 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
@@ -106,6 +106,6 @@ public static boolean isServiceSecuritySupported(IgniteProductVersion ver) {
                 return ver.compareToIgnoreTimestamp(v) >= 0;
         }
 
-        return ver.compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE[SERVICE_PERMISSIONS_SINCE.length]) >= 0;
+        return ver.compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE[SERVICE_PERMISSIONS_SINCE.length - 1]) >= 0;
     }
 }

From 3a1d560cd8741de9e7a6dd1110b42814d0ccff6b Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 6 Jun 2017 17:13:52 +0300
Subject: [PATCH 200/516] IGNITE-5259 Minor serialization fix

---
 .../ignite/internal/processors/security/SecurityUtils.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
index 306d8de58d87f..ca256c922f545 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityUtils.java
@@ -105,6 +105,6 @@ public static boolean isServiceSecuritySupported(IgniteProductVersion ver) {
                 return ver.compareToIgnoreTimestamp(v) >= 0;
         }
 
-        return ver.compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE[SERVICE_PERMISSIONS_SINCE.length]) >= 0;
+        return ver.compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE[SERVICE_PERMISSIONS_SINCE.length - 1]) >= 0;
     }
 }

From 56d4ce8a042238654ab96235d1a2969107b8881c Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Tue, 6 Jun 2017 17:39:33 +0300
Subject: [PATCH 201/516] GG-12244: Fixed a bug in GridH2IndexRangeRequest
 serialization mechanics.

---
 .../query/h2/twostep/msg/GridH2IndexRangeRequest.java           | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java
index b2548cc27a401..be188b04dd2a9 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java
@@ -174,6 +174,8 @@ public int batchLookupId() {
                 if (!writer.writeInt("segmentId", segmentId))
                     return false;
 
+                writer.incrementState();
+
             case 5:
                 if (!writer.writeInt("originSegId", originSegmentId))
                     return false;

From cfbe8da934741e76c8964af87671a38ec7b6c9a3 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 6 Jun 2017 19:15:59 +0300
Subject: [PATCH 202/516] IGNITE-5103 Rolled back due to test failings.

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  19 +--
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  57 +--------
 .../TcpDiscoveryClientSuspensionSelfTest.java | 116 ------------------
 3 files changed, 5 insertions(+), 187 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index e7bc81e5bc55f..b075dc1fb5289 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -367,7 +367,7 @@ else if (state == DISCONNECTED) {
                 else {
                     final GridFutureAdapter<Boolean> finalFut = fut;
 
-                    TimerTask task = new TimerTask() {
+                    timer.schedule(new TimerTask() {
                         @Override public void run() {
                             if (pingFuts.remove(nodeId, finalFut)) {
                                 if (ClientImpl.this.state == DISCONNECTED)
@@ -377,13 +377,7 @@ else if (state == DISCONNECTED) {
                                     finalFut.onDone(false);
                             }
                         }
-                    };
-
-                    try {
-                        timer.schedule(task, spi.netTimeout);
-                    } catch (IllegalStateException e) {
-                        return false;  // timer is cancelled because this client node is dying
-                    }
+                    }, spi.netTimeout);
 
                     sockWriter.sendMessage(new TcpDiscoveryClientPingRequest(getLocalNodeId(), nodeId));
                 }
@@ -819,13 +813,7 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
         U.warn(log, "Simulating client node failure: " + getLocalNodeId());
 
         U.interrupt(sockWriter);
-        U.interrupt(sockReader);
         U.interrupt(msgWorker);
-        try {
-            timer.cancel();
-        } catch (Throwable e) {
-            // No-op.
-        }
 
         U.join(sockWriter, log);
         U.join(
@@ -892,9 +880,6 @@ private class HeartbeatSender extends TimerTask {
                 msg.client(true);
 
                 sockWriter.sendMessage(msg);
-
-                if (log.isDebugEnabled())
-                    log.debug("*** Send heartbeat message from node:    "+msg.creatorNodeId());
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index a8feab41abd9a..c91954cb18c92 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6165,11 +6165,9 @@ private void processClientHeartbeatMessage(TcpDiscoveryClientHeartbeatMessage ms
             assert msg.client();
 
             ClientMessageWorker wrk = clientMsgWorkers.get(msg.creatorNodeId());
-            if (wrk != null) {
-                msg.verify(getLocalNodeId());
 
-                wrk.addMessage(msg);
-            }
+            if (wrk != null)
+                wrk.metrics(msg.metrics());
             else if (log.isDebugEnabled())
                 log.debug("Received heartbeat message from unknown client node: " + msg);
         }
@@ -6302,9 +6300,6 @@ private class StatisticsPrinter extends IgniteSpiThread {
     /**
      */
     private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAbstractMessage, byte[]>> {
-        /** minimal period of time which can be used as heartbeat timeout */
-        public static final int minHeartbeaTimeout = 10; // ms
-
         /** Node ID. */
         private final UUID clientNodeId;
 
@@ -6320,26 +6315,15 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
         /** */
         private IgniteProductVersion clientVer;
 
-        /**
-         * Period of time after which client node which does not send heartbeat messages, is considered dead.
-         * Measured in milliseconds
-         */
-        private long heartBeatTimeOut;
-
-        /** timestamp of the last received heartbeat message, in milliseconds */
-        private long lastHeartBeatTime;
-
         /**
          * @param sock Socket.
          * @param clientNodeId Node ID.
          */
         protected ClientMessageWorker(Socket sock, UUID clientNodeId) throws IOException {
-            super("tcp-disco-client-message-worker", Math.max(spi.getHeartbeatFrequency(), minHeartbeaTimeout));
+            super("tcp-disco-client-message-worker", 2000);
 
             this.sock = sock;
             this.clientNodeId = clientNodeId;
-            this.heartBeatTimeOut = spi.getHeartbeatFrequency() * spi.getMaxMissedClientHeartbeats();
-            this.lastHeartBeatTime=U.currentTimeMillis();
         }
 
         /**
@@ -6388,32 +6372,6 @@ void addMessage(TcpDiscoveryAbstractMessage msg, @Nullable byte[] msgBytes) {
                 log.debug("Message has been added to client queue: " + msg);
         }
 
-
-        /**
-         * Check the last time a heartbeat message received.
-         * In case of timeout, expel client node from the topology
-         */
-        @Override protected void noMessageLoop() {
-            long period = U.currentTimeMillis() - lastHeartBeatTime;
-
-            if (period >= heartBeatTimeOut) {
-                if (log.isInfoEnabled())
-                    log.info("Heartbeat timeout for node:" + clientNodeId + "; timeOut=" + heartBeatTimeOut + "; period=" + period);
-
-                TcpDiscoveryAbstractMessage msg = new TcpDiscoveryNodeLeftMessage(clientNodeId);
-
-                msg.senderNodeId(getLocalNodeId());
-
-                msgWorker.addMessage(msg);
-
-                clientMsgWorkers.remove(clientNodeId, this);
-
-                U.interrupt(this);
-
-                U.closeQuiet(sock);
-            }
-        }
-
         /** {@inheritDoc} */
         @Override protected void processMessage(T2<TcpDiscoveryAbstractMessage, byte[]> msgT) {
             boolean success = false;
@@ -6452,15 +6410,6 @@ else if (msgLog.isDebugEnabled())
                             spi.failureDetectionTimeout() : spi.getSocketTimeout());
                     }
                 }
-                else if (msg instanceof TcpDiscoveryClientHeartbeatMessage) {
-                    TcpDiscoveryClientHeartbeatMessage hbmsg = (TcpDiscoveryClientHeartbeatMessage)msg;
-
-                    if (log.isDebugEnabled()) // TODO turn to debug
-                        log.debug("###  Received heartbeat message from node:" + hbmsg.creatorNodeId());
-
-                    this.metrics = hbmsg.metrics();
-                    this.lastHeartBeatTime=U.currentTimeMillis();
-                }
                 else {
                     if (msgLog.isDebugEnabled())
                         msgLog.debug("Redirecting message to client [sock=" + sock + ", locNodeId="
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
deleted file mode 100644
index 9dc6e506cc74a..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.tcp;
-
-import java.util.Timer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Test for missed client heartbeats.
- */
-public class TcpDiscoveryClientSuspensionSelfTest extends GridCommonAbstractTest {
-    /** */
-    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        TcpDiscoverySpi disco = new TcpDiscoverySpi();
-
-        disco.setIpFinder(IP_FINDER);
-        disco.setHeartbeatFrequency(200);
-        disco.setMaxMissedClientHeartbeats(10);
-
-        cfg.setDiscoverySpi(disco);
-
-        return cfg;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        stopAllGrids();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testOneServer() throws Exception {
-        doTestClientSuspension(1);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testTwoServers() throws Exception {
-        doTestClientSuspension(2);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testThreeServers() throws Exception {
-        doTestClientSuspension(3);
-    }
-
-    /**
-     * @param serverCnt Servers count.
-     * @throws Exception If failed.
-     */
-    private void doTestClientSuspension(int serverCnt) throws Exception {
-        startGrids(serverCnt);
-
-        Ignition.setClientMode(true);
-
-        Ignite client = startGrid("client");
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(1, grid(i).cluster().forClients().nodes().size());
-
-        Thread.sleep(3000);
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(1, grid(i).cluster().forClients().nodes().size());
-
-        suspendClientHeartbeats(client);
-
-        Thread.sleep(3000);
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(0, grid(i).cluster().forClients().nodes().size());
-    }
-
-    /**
-     * @param client Client.
-     */
-    private void suspendClientHeartbeats(Ignite client) {
-        assert client.cluster().localNode().isClient();
-
-        ClientImpl impl = U.field(client.configuration().getDiscoverySpi(), "impl");
-
-        impl.simulateNodeFailure();
-
-        //Timer timer = U.field(impl, "timer"); timer.cancel(); -- client node successfully reconnects
-    }
-}

From 83307da08289c873c5c2eb02d5eb314018bc5c13 Mon Sep 17 00:00:00 2001
From: Ivan Veselovskiy <iveselovskiy@gridgain.com>
Date: Tue, 6 Jun 2017 16:56:09 +0300
Subject: [PATCH 203/516] IGNITE-5410: Fixed assertion in HadoopDataOutStream.
 This closes #2084.

(cherry picked from commit d2bf961)
---
 .../processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
index acc9be62cf677..655346eaae93a 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopOffheapBuffer.java
@@ -92,7 +92,7 @@ public void pointer(long ptr) {
      * @return Old position pointer or {@code 0} if move goes beyond the end of the buffer.
      */
     public long move(long size) {
-        assert size > 0 : size;
+        assert size >= 0 : size;
 
         long oldPos = posPtr;
         long newPos = oldPos + size;

From e95626d609ee225918b49653b7981b180e5d4e49 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 1 Jun 2017 19:56:34 +0300
Subject: [PATCH 204/516] SSL fix

(cherry picked from commit 95d5595)
---
 .../ignite/spi/communication/tcp/TcpCommunicationSpi.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 60d97348d42cf..dbad401e9d037 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3198,10 +3198,10 @@ else if (log.isDebugEnabled())
                             assert sslHnd != null;
 
                             buf = ByteBuffer.allocate(1000);
+                            buf.order(ByteOrder.nativeOrder());
 
                             ByteBuffer decode = ByteBuffer.allocate(2 * buf.capacity());
-
-                            buf.order(ByteOrder.nativeOrder());
+                            decode.order(ByteOrder.nativeOrder());
 
                             for (int i = 0; i < 9; ) {
                                 int read = ch.read(buf);

From 340204637a03e5533685f1b11ca65c9121f6e193 Mon Sep 17 00:00:00 2001
From: Alexei Kaigorodov <alexei.kaigorodov@gmail.com>
Date: Thu, 8 Jun 2017 19:37:40 +0300
Subject: [PATCH 205/516] IGNITE-5103 Rolled back due to test failings. (#69)

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  19 +--
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  57 +--------
 .../TcpDiscoveryClientSuspensionSelfTest.java | 116 ------------------
 3 files changed, 5 insertions(+), 187 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index f4262f197cb63..a5a4e88990c4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -366,7 +366,7 @@ else if (state == DISCONNECTED) {
                 else {
                     final GridFutureAdapter<Boolean> finalFut = fut;
 
-                    TimerTask task = new TimerTask() {
+                    timer.schedule(new TimerTask() {
                         @Override public void run() {
                             if (pingFuts.remove(nodeId, finalFut)) {
                                 if (ClientImpl.this.state == DISCONNECTED)
@@ -376,13 +376,7 @@ else if (state == DISCONNECTED) {
                                     finalFut.onDone(false);
                             }
                         }
-                    };
-
-                    try {
-                        timer.schedule(task, spi.netTimeout);
-                    } catch (IllegalStateException e) {
-                        return false;  // timer is cancelled because this client node is dying
-                    }
+                    }, spi.netTimeout);
 
                     sockWriter.sendMessage(new TcpDiscoveryClientPingRequest(getLocalNodeId(), nodeId));
                 }
@@ -815,13 +809,7 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
         U.warn(log, "Simulating client node failure: " + getLocalNodeId());
 
         U.interrupt(sockWriter);
-        U.interrupt(sockReader);
         U.interrupt(msgWorker);
-        try {
-            timer.cancel();
-        } catch (Throwable e) {
-            // No-op.
-        }
 
         U.join(sockWriter, log);
         U.join(
@@ -888,9 +876,6 @@ private class HeartbeatSender extends TimerTask {
                 msg.client(true);
 
                 sockWriter.sendMessage(msg);
-
-                if (log.isDebugEnabled())
-                    log.debug("*** Send heartbeat message from node:    "+msg.creatorNodeId());
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 211265b712524..eef8f45a3064e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6164,11 +6164,9 @@ private void processClientHeartbeatMessage(TcpDiscoveryClientHeartbeatMessage ms
             assert msg.client();
 
             ClientMessageWorker wrk = clientMsgWorkers.get(msg.creatorNodeId());
-            if (wrk != null) {
-                msg.verify(getLocalNodeId());
 
-                wrk.addMessage(msg);
-            }
+            if (wrk != null)
+                wrk.metrics(msg.metrics());
             else if (log.isDebugEnabled())
                 log.debug("Received heartbeat message from unknown client node: " + msg);
         }
@@ -6301,9 +6299,6 @@ private class StatisticsPrinter extends IgniteSpiThread {
     /**
      */
     private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAbstractMessage, byte[]>> {
-        /** minimal period of time which can be used as heartbeat timeout */
-        public static final int minHeartbeaTimeout = 10; // ms
-
         /** Node ID. */
         private final UUID clientNodeId;
 
@@ -6319,26 +6314,15 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
         /** */
         private IgniteProductVersion clientVer;
 
-        /**
-         * Period of time after which client node which does not send heartbeat messages, is considered dead.
-         * Measured in milliseconds
-         */
-        private long heartBeatTimeOut;
-
-        /** timestamp of the last received heartbeat message, in milliseconds */
-        private long lastHeartBeatTime;
-
         /**
          * @param sock Socket.
          * @param clientNodeId Node ID.
          */
         protected ClientMessageWorker(Socket sock, UUID clientNodeId) throws IOException {
-            super("tcp-disco-client-message-worker", Math.max(spi.getHeartbeatFrequency(), minHeartbeaTimeout));
+            super("tcp-disco-client-message-worker", 2000);
 
             this.sock = sock;
             this.clientNodeId = clientNodeId;
-            this.heartBeatTimeOut = spi.getHeartbeatFrequency() * spi.getMaxMissedClientHeartbeats();
-            this.lastHeartBeatTime=U.currentTimeMillis();
         }
 
         /**
@@ -6387,32 +6371,6 @@ void addMessage(TcpDiscoveryAbstractMessage msg, @Nullable byte[] msgBytes) {
                 log.debug("Message has been added to client queue: " + msg);
         }
 
-
-        /**
-         * Check the last time a heartbeat message received.
-         * In case of timeout, expel client node from the topology
-         */
-        @Override protected void noMessageLoop() {
-            long period = U.currentTimeMillis() - lastHeartBeatTime;
-
-            if (period >= heartBeatTimeOut) {
-                if (log.isInfoEnabled())
-                    log.info("Heartbeat timeout for node:" + clientNodeId + "; timeOut=" + heartBeatTimeOut + "; period=" + period);
-
-                TcpDiscoveryAbstractMessage msg = new TcpDiscoveryNodeLeftMessage(clientNodeId);
-
-                msg.senderNodeId(getLocalNodeId());
-
-                msgWorker.addMessage(msg);
-
-                clientMsgWorkers.remove(clientNodeId, this);
-
-                U.interrupt(this);
-
-                U.closeQuiet(sock);
-            }
-        }
-
         /** {@inheritDoc} */
         @Override protected void processMessage(T2<TcpDiscoveryAbstractMessage, byte[]> msgT) {
             boolean success = false;
@@ -6451,15 +6409,6 @@ else if (msgLog.isDebugEnabled())
                             spi.failureDetectionTimeout() : spi.getSocketTimeout());
                     }
                 }
-                else if (msg instanceof TcpDiscoveryClientHeartbeatMessage) {
-                    TcpDiscoveryClientHeartbeatMessage hbmsg = (TcpDiscoveryClientHeartbeatMessage)msg;
-
-                    if (log.isDebugEnabled()) // TODO turn to debug
-                        log.debug("###  Received heartbeat message from node:" + hbmsg.creatorNodeId());
-
-                    this.metrics = hbmsg.metrics();
-                    this.lastHeartBeatTime=U.currentTimeMillis();
-                }
                 else {
                     if (msgLog.isDebugEnabled())
                         msgLog.debug("Redirecting message to client [sock=" + sock + ", locNodeId="
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
deleted file mode 100644
index 9dc6e506cc74a..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.tcp;
-
-import java.util.Timer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Test for missed client heartbeats.
- */
-public class TcpDiscoveryClientSuspensionSelfTest extends GridCommonAbstractTest {
-    /** */
-    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        TcpDiscoverySpi disco = new TcpDiscoverySpi();
-
-        disco.setIpFinder(IP_FINDER);
-        disco.setHeartbeatFrequency(200);
-        disco.setMaxMissedClientHeartbeats(10);
-
-        cfg.setDiscoverySpi(disco);
-
-        return cfg;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        stopAllGrids();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testOneServer() throws Exception {
-        doTestClientSuspension(1);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testTwoServers() throws Exception {
-        doTestClientSuspension(2);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testThreeServers() throws Exception {
-        doTestClientSuspension(3);
-    }
-
-    /**
-     * @param serverCnt Servers count.
-     * @throws Exception If failed.
-     */
-    private void doTestClientSuspension(int serverCnt) throws Exception {
-        startGrids(serverCnt);
-
-        Ignition.setClientMode(true);
-
-        Ignite client = startGrid("client");
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(1, grid(i).cluster().forClients().nodes().size());
-
-        Thread.sleep(3000);
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(1, grid(i).cluster().forClients().nodes().size());
-
-        suspendClientHeartbeats(client);
-
-        Thread.sleep(3000);
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(0, grid(i).cluster().forClients().nodes().size());
-    }
-
-    /**
-     * @param client Client.
-     */
-    private void suspendClientHeartbeats(Ignite client) {
-        assert client.cluster().localNode().isClient();
-
-        ClientImpl impl = U.field(client.configuration().getDiscoverySpi(), "impl");
-
-        impl.simulateNodeFailure();
-
-        //Timer timer = U.field(impl, "timer"); timer.cancel(); -- client node successfully reconnects
-    }
-}

From f3f726e9059e492573dc5125fd5edb5d2f71e9d3 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 13 Jun 2017 14:11:17 +0300
Subject: [PATCH 206/516] IGNITE-4196: Added means to specify port for H2 debug
 console. This closes #1486.

(cherry picked from commit b246260)
---
 .../java/org/apache/ignite/IgniteSystemProperties.java    | 8 ++++++++
 .../internal/processors/query/h2/IgniteH2Indexing.java    | 7 ++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index e933d83dda840..88aeae6bfccd5 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -326,6 +326,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_H2_DEBUG_CONSOLE = "IGNITE_H2_DEBUG_CONSOLE";
 
+    /**
+     * This property allows to specify user defined port which H2 indexing SPI will use
+     * to start H2 debug console on. If this property is not set or set to 0, H2 debug
+     * console will use system-provided dynamic port.
+     * This property is only relevant when {@link #IGNITE_H2_DEBUG_CONSOLE} property is set.
+     */
+    public static final String IGNITE_H2_DEBUG_CONSOLE_PORT = "IGNITE_H2_DEBUG_CONSOLE_PORT";
+
     /**
      * If this property is set to {@code true} then shared memory space native debug will be enabled.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 8c5c2a364cabf..66fb7ae94f5be 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -176,6 +176,7 @@
 import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE_PORT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_CLEANUP_PERIOD;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_THREAD_USAGE_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.getInteger;
@@ -1797,11 +1798,15 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
             if (getString(IGNITE_H2_DEBUG_CONSOLE) != null) {
                 Connection c = DriverManager.getConnection(dbUrl);
 
+                int port = getInteger(IGNITE_H2_DEBUG_CONSOLE_PORT, 0);
+
                 WebServer webSrv = new WebServer();
-                Server web = new Server(webSrv, "-webPort", "0");
+                Server web = new Server(webSrv, "-webPort", Integer.toString(port));
                 web.start();
                 String url = webSrv.addSession(c);
 
+                U.quietAndInfo(log, "H2 debug console URL: " + url);
+
                 try {
                     Server.openBrowser(url);
                 }

From c2c237d1222557d3e6b35d9a51a61a4c78e56782 Mon Sep 17 00:00:00 2001
From: Sergey Kalashnikov <skalashnikov@gridgain.com>
Date: Fri, 3 Feb 2017 11:41:14 +0300
Subject: [PATCH 207/516] IGNITE-4196: Added means to specify port for H2 debug
 console. This closes #1486.

(cherry picked from commit b246260)
---
 .../java/org/apache/ignite/IgniteSystemProperties.java    | 8 ++++++++
 .../internal/processors/query/h2/IgniteH2Indexing.java    | 8 +++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 6827e0c248290..f35e9e620070a 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -320,6 +320,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_H2_DEBUG_CONSOLE = "IGNITE_H2_DEBUG_CONSOLE";
 
+    /**
+     * This property allows to specify user defined port which H2 indexing SPI will use
+     * to start H2 debug console on. If this property is not set or set to 0, H2 debug
+     * console will use system-provided dynamic port.
+     * This property is only relevant when {@link #IGNITE_H2_DEBUG_CONSOLE} property is set.
+     */
+    public static final String IGNITE_H2_DEBUG_CONSOLE_PORT = "IGNITE_H2_DEBUG_CONSOLE_PORT";
+
     /**
      * If this property is set to {@code true} then shared memory space native debug will be enabled.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 62b47b85c6507..8b2993f263cf7 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -171,8 +171,10 @@
 import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_DEBUG_CONSOLE_PORT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_CLEANUP_PERIOD;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_H2_INDEXING_CACHE_THREAD_USAGE_TIMEOUT;
+import static org.apache.ignite.IgniteSystemProperties.getInteger;
 import static org.apache.ignite.IgniteSystemProperties.getString;
 import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL;
 import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
@@ -1751,11 +1753,15 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
             if (getString(IGNITE_H2_DEBUG_CONSOLE) != null) {
                 Connection c = DriverManager.getConnection(dbUrl);
 
+                int port = getInteger(IGNITE_H2_DEBUG_CONSOLE_PORT, 0);
+
                 WebServer webSrv = new WebServer();
-                Server web = new Server(webSrv, "-webPort", "0");
+                Server web = new Server(webSrv, "-webPort", Integer.toString(port));
                 web.start();
                 String url = webSrv.addSession(c);
 
+                U.quietAndInfo(log, "H2 debug console URL: " + url);
+
                 try {
                     Server.openBrowser(url);
                 }

From 4a8f295f2f2f34e8472b1d1320f03744135b2504 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 13 Jun 2017 19:47:00 +0300
Subject: [PATCH 208/516] IGNITE-5478: ODBC: SQLNumParams now returns number of
 required parameters.

(cherry picked from commit b1c56a1)
---
 .../cpp/odbc-test/src/queries_test.cpp        | 30 +++++++++++++++++
 .../cpp/odbc/include/ignite/odbc/statement.h  | 13 ++++++--
 modules/platforms/cpp/odbc/src/odbc.cpp       |  7 +++-
 modules/platforms/cpp/odbc/src/statement.cpp  | 33 +++++++++++++++++--
 4 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 52b885d1d2298..6dc5d4f63cf3c 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -273,6 +273,25 @@ struct QueriesTestSuiteFixture
         BOOST_CHECK(ret == SQL_NO_DATA);
     }
 
+    void CheckParamsNum(const std::string& req, SQLSMALLINT expectedParamsNum)
+    {
+        std::vector<SQLCHAR> req0(req.begin(), req.end());
+
+        SQLRETURN ret = SQLPrepare(stmt, &req0[0], static_cast<SQLINTEGER>(req0.size()));
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        SQLSMALLINT paramsNum = -1;
+
+        ret = SQLNumParams(stmt, &paramsNum);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECK_EQUAL(paramsNum, expectedParamsNum);
+    }
+
     int CountRows(SQLHSTMT stmt)
     {
         int res = 0;
@@ -1342,4 +1361,15 @@ BOOST_AUTO_TEST_CASE(TestInsertMergeSelect)
     BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
 }
 
+BOOST_AUTO_TEST_CASE(TestParamsNum)
+{
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+
+    CheckParamsNum("SELECT * FROM TestType", 0);
+    CheckParamsNum("SELECT * FROM TestType WHERE _key=?", 1);
+    CheckParamsNum("SELECT * FROM TestType WHERE _key=? AND _val=?", 2);
+    CheckParamsNum("INSERT INTO TestType(_key, strField) VALUES(1, 'some')", 0);
+    CheckParamsNum("INSERT INTO TestType(_key, strField) VALUES(?, ?)", 2);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
index db566609d1b86..1ee56197ce81b 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
@@ -137,11 +137,11 @@ namespace ignite
             void GetAttribute(int attr, void* buf, SQLINTEGER bufLen, SQLINTEGER *valueLen);
 
             /**
-             * Get number of binded parameters.
+             * Get number parameters required by the prepared statement.
              *
-             * @return Number of binded parameters.
+             * @param paramNum Number of parameters.
              */
-            uint16_t GetParametersNumber();
+            void GetParametersNumber(uint16_t& paramNum);
 
             /**
              * Set parameter binding offset pointer.
@@ -394,6 +394,13 @@ namespace ignite
              */
             SqlResult InternalGetAttribute(int attr, void* buf, SQLINTEGER bufLen, SQLINTEGER* valueLen);
 
+            /**
+             * Get number parameters required by the prepared statement.
+             *
+             * @param paramNum Number of parameters.
+             */
+            SqlResult InternalGetParametersNumber(uint16_t& paramNum);
+
             /**
              * Get value of the column in the result set.
              *
diff --git a/modules/platforms/cpp/odbc/src/odbc.cpp b/modules/platforms/cpp/odbc/src/odbc.cpp
index 0b18a116f4993..caf31d91860b1 100644
--- a/modules/platforms/cpp/odbc/src/odbc.cpp
+++ b/modules/platforms/cpp/odbc/src/odbc.cpp
@@ -929,7 +929,12 @@ namespace ignite
             return SQL_INVALID_HANDLE;
 
         if (paramCnt)
-            *paramCnt = static_cast<SQLSMALLINT>(statement->GetParametersNumber());
+        {
+            uint16_t paramNum = 0;
+            statement->GetParametersNumber(paramNum);
+
+            *paramCnt = static_cast<SQLSMALLINT>(paramNum);
+        }
 
         return statement->GetDiagnosticRecords().GetReturnCode();
     }
diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp
index 02c6dd972154b..9aca8c9164209 100644
--- a/modules/platforms/cpp/odbc/src/statement.cpp
+++ b/modules/platforms/cpp/odbc/src/statement.cpp
@@ -296,11 +296,38 @@ namespace ignite
             return SQL_RESULT_SUCCESS;
         }
 
-        uint16_t Statement::GetParametersNumber()
+        void Statement::GetParametersNumber(uint16_t& paramNum)
         {
-            IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
+            IGNITE_ODBC_API_CALL(InternalGetParametersNumber(paramNum));
+        }
+
+        SqlResult Statement::InternalGetParametersNumber(uint16_t& paramNum)
+        {
+            if (!currentQuery.get())
+            {
+                AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query is not prepared.");
+
+                return SQL_RESULT_ERROR;
+            }
+
+            if (currentQuery->GetType() != query::Query::DATA)
+            {
+                paramNum = 0;
 
-            return static_cast<uint16_t>(paramBindings.size());
+                return SQL_RESULT_SUCCESS;
+            }
+
+            if (paramTypes.empty())
+            {
+                SqlResult res = UpdateParamsMeta();
+
+                if (res != SQL_RESULT_SUCCESS)
+                    return res;
+            }
+
+            paramNum = static_cast<uint16_t>(paramTypes.size());
+
+            return SQL_RESULT_SUCCESS;
         }
 
         void Statement::SetParamBindOffsetPtr(int* ptr)

From a2a4ec1ee9794cb542f146a07c6c67002cad444e Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Wed, 14 Jun 2017 12:16:43 +0300
Subject: [PATCH 209/516] IGNITE-5478: Fix for cherry pick

---
 modules/platforms/cpp/odbc-test/src/queries_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 6dc5d4f63cf3c..3a3858802459e 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -1363,7 +1363,7 @@ BOOST_AUTO_TEST_CASE(TestInsertMergeSelect)
 
 BOOST_AUTO_TEST_CASE(TestParamsNum)
 {
-    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
 
     CheckParamsNum("SELECT * FROM TestType", 0);
     CheckParamsNum("SELECT * FROM TestType WHERE _key=?", 1);

From d268b32cb252a5f06887d2b803d27ddc20ded95f Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Fri, 16 Jun 2017 12:27:35 +0300
Subject: [PATCH 210/516] IGNITE-4370: Implemented writing of batch of
 parameters for ODBC.

(cherry picked from commit c10be5780589cc84e7929e234e4411d515166e0b)
---
 .../processors/odbc/OdbcMessageParser.java    |  57 ++-
 .../processors/odbc/OdbcNioListener.java      |   3 +-
 .../odbc/OdbcQueryExecuteBatchRequest.java    |  95 ++++
 .../odbc/OdbcQueryExecuteBatchResult.java     |  75 ++++
 .../odbc/OdbcQueryExecuteRequest.java         |   6 +-
 .../odbc/OdbcQueryExecuteResult.java          |   2 +-
 .../internal/processors/odbc/OdbcRequest.java |   3 +
 .../processors/odbc/OdbcRequestHandler.java   | 106 ++++-
 .../cpp/core-test/src/test_utils.cpp          |   0
 .../src/application_data_buffer_test.cpp      | 146 ++-----
 .../cpp/odbc-test/src/column_test.cpp         |  12 +-
 .../cpp/odbc-test/src/queries_test.cpp        | 386 ++++++++++++++++-
 .../platforms/cpp/odbc-test/src/row_test.cpp  |   9 +-
 modules/platforms/cpp/odbc/Makefile.am        |   3 +
 .../platforms/cpp/odbc/include/Makefile.am    |   2 +
 .../ignite/odbc/app/application_data_buffer.h |  38 +-
 .../odbc/include/ignite/odbc/app/parameter.h  |  20 +-
 .../include/ignite/odbc/app/parameter_set.h   | 268 ++++++++++++
 .../odbc/include/ignite/odbc/common_types.h   |   8 +-
 .../ignite/odbc/diagnostic/diagnosable.h      |   4 +-
 .../cpp/odbc/include/ignite/odbc/message.h    | 408 +++++++-----------
 .../include/ignite/odbc/query/batch_query.h   | 160 +++++++
 .../include/ignite/odbc/query/data_query.h    |  20 +-
 .../odbc/include/ignite/odbc/query/query.h    |   3 +
 .../ignite/odbc/query/type_info_query.h       |   2 +-
 .../cpp/odbc/include/ignite/odbc/statement.h  |  59 ++-
 .../cpp/odbc/project/vs/odbc.vcxproj          |   5 +
 .../cpp/odbc/project/vs/odbc.vcxproj.filters  |  15 +
 .../odbc/src/app/application_data_buffer.cpp  |  94 +++-
 .../platforms/cpp/odbc/src/app/parameter.cpp  |  13 +-
 .../cpp/odbc/src/app/parameter_set.cpp        | 242 +++++++++++
 .../cpp/odbc/src/config/connection_info.cpp   |  68 ++-
 .../odbc/src/diagnostic/diagnostic_record.cpp |  12 +
 modules/platforms/cpp/odbc/src/message.cpp    | 366 ++++++++++++++++
 modules/platforms/cpp/odbc/src/odbc.cpp       |  23 +-
 .../cpp/odbc/src/query/batch_query.cpp        | 197 +++++++++
 .../cpp/odbc/src/query/data_query.cpp         |  15 +-
 modules/platforms/cpp/odbc/src/statement.cpp  | 260 +++++++----
 38 files changed, 2621 insertions(+), 584 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java
 create mode 100644 modules/platforms/cpp/core-test/src/test_utils.cpp
 create mode 100644 modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
 create mode 100644 modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
 create mode 100644 modules/platforms/cpp/odbc/src/app/parameter_set.cpp
 create mode 100644 modules/platforms/cpp/odbc/src/message.cpp
 create mode 100644 modules/platforms/cpp/odbc/src/query/batch_query.cpp

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java
index 7b863d6b2af21..0cb89d71f626b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
 
 import java.util.Collection;
+import org.jetbrains.annotations.NotNull;
 
 /**
  * ODBC message parser.
@@ -99,23 +100,37 @@ public OdbcRequest decode(byte[] msg) {
         }
 
         OdbcRequest res;
-
+        
         switch (cmd) {
             case OdbcRequest.EXECUTE_SQL_QUERY: {
                 String cache = reader.readString();
                 String sql = reader.readString();
-                int argsNum = reader.readInt();
-
-                Object[] params = new Object[argsNum];
+                int paramNum = reader.readInt();
 
-                for (int i = 0; i < argsNum; ++i)
-                    params[i] = reader.readObjectDetached();
+                Object[] params = readParameterRow(reader, paramNum);
 
                 res = new OdbcQueryExecuteRequest(cache, sql, params);
 
                 break;
             }
 
+            case OdbcRequest.QRY_EXEC_BATCH: {
+                String schema = reader.readString();
+                String sql = reader.readString();
+                int paramRowLen = reader.readInt();
+                int rowNum = reader.readInt();
+                boolean last = reader.readBoolean();
+
+                Object[][] params = new Object[rowNum][];
+
+                for (int i = 0; i < rowNum; ++i)
+                    params[i] = readParameterRow(reader, paramRowLen);
+
+                res = new OdbcQueryExecuteBatchRequest(schema, sql, last, params);
+
+                break;
+            }
+
             case OdbcRequest.FETCH_SQL_QUERY: {
                 long queryId = reader.readLong();
                 int pageSize = reader.readInt();
@@ -170,6 +185,21 @@ public OdbcRequest decode(byte[] msg) {
         return res;
     }
 
+    /**
+     * Read row of parameters using reader.
+     * @param reader reader
+     * @param paramNum Number of parameters in a row
+     * @return Parameters array.
+     */
+    @NotNull private static Object[] readParameterRow(BinaryReaderExImpl reader, int paramNum) {
+        Object[] params = new Object[paramNum];
+
+        for (int i = 0; i < paramNum; ++i)
+            params[i] = reader.readObjectDetached();
+
+        return params;
+    }
+
     /**
      * Encode OdbcResponse to byte array.
      *
@@ -218,9 +248,9 @@ else if (res0 instanceof OdbcQueryExecuteResult) {
             OdbcQueryExecuteResult res = (OdbcQueryExecuteResult) res0;
 
             if (log.isDebugEnabled())
-                log.debug("Resulting query ID: " + res.getQueryId());
+                log.debug("Resulting query ID: " + res.queryId());
 
-            writer.writeLong(res.getQueryId());
+            writer.writeLong(res.queryId());
 
             Collection<OdbcColumnMeta> metas = res.getColumnsMetadata();
 
@@ -231,6 +261,17 @@ else if (res0 instanceof OdbcQueryExecuteResult) {
             for (OdbcColumnMeta meta : metas)
                 meta.write(writer);
         }
+        else if (res0 instanceof OdbcQueryExecuteBatchResult) {
+            OdbcQueryExecuteBatchResult res = (OdbcQueryExecuteBatchResult) res0;
+
+            writer.writeBoolean(res.errorMessage() == null);
+            writer.writeLong(res.rowsAffected());
+
+            if (res.errorMessage() != null) {
+                writer.writeLong(res.errorSetIdx());
+                writer.writeString(res.errorMessage());
+            }
+        }
         else if (res0 instanceof OdbcQueryFetchResult) {
             OdbcQueryFetchResult res = (OdbcQueryFetchResult) res0;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java
index 378e5f25ebebc..a68cf888af48b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.internal.util.nio.GridNioServerListenerAdapter;
 import org.apache.ignite.internal.util.nio.GridNioSession;
 import org.apache.ignite.internal.util.nio.GridNioSessionMetaKey;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.concurrent.atomic.AtomicLong;
@@ -100,7 +101,7 @@ public OdbcNioListener(GridKernalContext ctx, GridSpinBusyLock busyLock, int max
             req = parser.decode(msg);
         }
         catch (Exception e) {
-            log.error("Failed to parse message [id=" + reqId + ", err=" + e + ']');
+            U.error(log, "Failed to parse message [id=" + reqId + ']', e);
 
             ses.close();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java
new file mode 100644
index 0000000000000..0ace947126349
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java
@@ -0,0 +1,95 @@
+package org.apache.ignite.internal.processors.odbc;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * ODBC query execute request with the batch of parameters.
+ */
+public class OdbcQueryExecuteBatchRequest extends OdbcRequest {
+    /** Schema. */
+    @GridToStringInclude(sensitive = true)
+    private final String schema;
+
+    /** Sql query. */
+    @GridToStringInclude(sensitive = true)
+    private final String sqlQry;
+
+    /** Last param page flag. */
+    private final boolean last;
+
+    /** Sql query arguments. */
+    @GridToStringExclude
+    private final Object[][] args;
+
+    /**
+     * @param schema Schema.
+     * @param sqlQry SQL query.
+     * @param last Last page flag.
+     * @param args Arguments list.
+     */
+    public OdbcQueryExecuteBatchRequest(@Nullable String schema, String sqlQry, boolean last, Object[][] args) {
+        super(QRY_EXEC_BATCH);
+
+        assert sqlQry != null : "SQL query should not be null";
+        assert args != null : "Parameters should not be null";
+
+        this.schema = schema;
+        this.sqlQry = sqlQry;
+        this.last = last;
+        this.args = args;
+    }
+
+    /**
+     * @return Sql query.
+     */
+    public String sqlQuery() {
+        return sqlQry;
+    }
+
+    /**
+     * @return Sql query arguments.
+     */
+    public Object[][] arguments() {
+        return args;
+    }
+
+    /**
+     * @return Schema.
+     */
+    @Nullable
+    public String schema() {
+        return schema;
+    }
+
+    /**
+     * @return Last page flag.
+     */
+    public boolean last() {
+        return last;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(OdbcQueryExecuteBatchRequest.class, this, "args", args, true);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java
new file mode 100644
index 0000000000000..6fc3873dea267
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.odbc;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * ODBC query execute with batch of parameters result.
+ */
+public class OdbcQueryExecuteBatchResult {
+    /** Rows affected. */
+    private final long rowsAffected;
+
+    /** Index of the set which caused an error. */
+    private final long errorSetIdx;
+
+    /** Error message. */
+    private final String errorMessage;
+
+    /**
+     * @param rowsAffected Number of rows affected by the query.
+     */
+    public OdbcQueryExecuteBatchResult(long rowsAffected) {
+        this.rowsAffected = rowsAffected;
+        this.errorSetIdx = -1;
+        this.errorMessage = null;
+    }
+
+    /**
+     * @param rowsAffected Number of rows affected by the query.
+     * @param errorSetIdx Sets processed.
+     * @param errorMessage Error message.
+     */
+    public OdbcQueryExecuteBatchResult(long rowsAffected, long errorSetIdx, String errorMessage) {
+        this.rowsAffected = rowsAffected;
+        this.errorSetIdx = errorSetIdx;
+        this.errorMessage = errorMessage;
+    }
+
+    /**
+     * @return Number of rows affected by the query.
+     */
+    public long rowsAffected() {
+        return rowsAffected;
+    }
+
+    /**
+     * @return Index of the set which caused an error or -1 if no error occurred.
+     */
+    public long errorSetIdx() {
+        return errorSetIdx;
+    }
+
+    /**
+     * @return Error message.
+     */
+    @Nullable public String errorMessage() {
+        return errorMessage;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
index c0d1c601c92b7..029135cd52e80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
@@ -42,10 +42,12 @@ public class OdbcQueryExecuteRequest extends OdbcRequest {
      * @param sqlQry SQL query.
      * @param args Arguments list.
      */
-    public OdbcQueryExecuteRequest(String cacheName, String sqlQry, Object[] args) {
+    public OdbcQueryExecuteRequest(@Nullable String cacheName, String sqlQry, Object[] args) {
         super(EXECUTE_SQL_QUERY);
 
-        this.cacheName = cacheName.isEmpty() ? null : cacheName;
+        assert sqlQry != null : "SQL query should not be null";
+
+        this.cacheName = cacheName;
         this.sqlQry = sqlQry;
         this.args = args;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java
index a4038123d2d4d..efa432b615614 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java
@@ -41,7 +41,7 @@ public OdbcQueryExecuteResult(long queryId, Collection<OdbcColumnMeta> columnsMe
     /**
      * @return Query ID.
      */
-    public long getQueryId() {
+    public long queryId() {
         return queryId;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java
index f056fedf079bd..3c0c12e09cd28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java
@@ -42,6 +42,9 @@ public class OdbcRequest {
     /** Get parameters meta. */
     public static final int GET_PARAMS_META = 7;
 
+    /** Execute sql query with the batch of parameters. */
+    public static final int QRY_EXEC_BATCH = 8;
+
     /** Command. */
     private final int cmd;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
index c6b41d2d75d36..06430834b2d8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.internal.processors.odbc;
 
 import org.apache.ignite.IgniteCache;
+
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
@@ -36,7 +38,11 @@
 import java.sql.ParameterMetaData;
 import java.sql.PreparedStatement;
 import java.sql.Types;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -104,6 +110,9 @@ public OdbcResponse handle(long reqId, OdbcRequest req) {
                 case HANDSHAKE:
                     return performHandshake(reqId, (OdbcHandshakeRequest)req);
 
+                case QRY_EXEC_BATCH:
+                    return executeBatchQuery((OdbcQueryExecuteBatchRequest)req);
+
                 case EXECUTE_SQL_QUERY:
                     return executeQuery(reqId, (OdbcQueryExecuteRequest)req);
 
@@ -229,6 +238,101 @@ private OdbcResponse executeQuery(long reqId, OdbcQueryExecuteRequest req) {
         }
     }
 
+    /**
+     * {@link OdbcQueryExecuteBatchRequest} command handler.
+     *
+     * @param req Execute query request.
+     * @return Response.
+     */
+    private OdbcResponse executeBatchQuery(OdbcQueryExecuteBatchRequest req) {
+        long rowsAffected = 0;
+        int currentSet = 0;
+
+        try {
+            String sql = OdbcEscapeUtils.parse(req.sqlQuery());
+
+            if (log.isDebugEnabled())
+                log.debug("ODBC query parsed [original=" + req.sqlQuery() + ", parsed=" + sql + ']');
+
+            SqlFieldsQuery qry = new SqlFieldsQuery(sql);
+
+            qry.setDistributedJoins(distributedJoins);
+            qry.setEnforceJoinOrder(enforceJoinOrder);
+
+            IgniteCache<Object, Object> cache0 = ctx.grid().cache(req.schema());
+
+            if (cache0 == null)
+                return new OdbcResponse(OdbcResponse.STATUS_FAILED,
+                        "Cache doesn't exist (did you configure it?): " + req.schema());
+
+            IgniteCache<Object, Object> cache = cache0.withKeepBinary();
+
+            if (cache == null)
+                return new OdbcResponse(OdbcResponse.STATUS_FAILED,
+                        "Can not get cache with keep binary: " + req.schema());
+
+            Object[][] paramSet = req.arguments();
+
+            if (paramSet.length <= 0)
+                throw new IgniteException("Batch execute request with non-positive batch length. [len="
+                        + paramSet.length + ']');
+
+            // Getting meta and do the checks for the first execution.
+            qry.setArgs(paramSet[0]);
+
+            QueryCursorImpl<List<?>> qryCur = (QueryCursorImpl<List<?>>)cache.query(qry);
+
+            if (qryCur.isQuery())
+                throw new IgniteException("Batching of parameters only supported for DML statements. [query=" +
+                        req.sqlQuery() + ']');
+
+            rowsAffected += getRowsAffected(qryCur);
+
+            for (currentSet = 1; currentSet < paramSet.length; ++currentSet)
+            {
+                qry.setArgs(paramSet[currentSet]);
+
+                QueryCursor<List<?>> cur = cache.query(qry);
+
+                rowsAffected += getRowsAffected(cur);
+            }
+
+            OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected);
+
+            return new OdbcResponse(res);
+        }
+        catch (Exception e) {
+            U.error(log, "Failed to execute SQL query [req=" + req + ']', e);
+
+            OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected, currentSet,
+                    e.getMessage());
+
+            return new OdbcResponse(res);
+        }
+    }
+
+    /**
+     * Get affected rows for DML statement.
+     * @param qryCur Cursor.
+     * @return Number of table rows affected.
+     */
+    private static long getRowsAffected(QueryCursor<List<?>> qryCur) {
+        Iterator<List<?>> iter = qryCur.iterator();
+
+        if (iter.hasNext()) {
+            List<?> res = iter.next();
+
+            if (res.size() > 0) {
+                Long affected = (Long) res.get(0);
+
+                if (affected != null)
+                    return affected;
+            }
+        }
+
+        return 0;
+    }
+
     /**
      * {@link OdbcQueryCloseRequest} command handler.
      *
diff --git a/modules/platforms/cpp/core-test/src/test_utils.cpp b/modules/platforms/cpp/core-test/src/test_utils.cpp
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp b/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp
index fe502950a2898..fcc744ac89107 100644
--- a/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp
@@ -44,9 +44,8 @@ BOOST_AUTO_TEST_CASE(TestPutIntToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     appBuf.PutInt8(12);
     BOOST_CHECK(!strcmp(buffer, "12"));
@@ -77,9 +76,8 @@ BOOST_AUTO_TEST_CASE(TestPutFloatToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     appBuf.PutFloat(12.42f);
     BOOST_CHECK(!strcmp(buffer, "12.42"));
@@ -102,9 +100,8 @@ BOOST_AUTO_TEST_CASE(TestPutGuidToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     ignite::Guid guid(0x1da1ef8f39ff4d62ULL, 0x8b72e8e9f3371801ULL);
 
@@ -119,7 +116,7 @@ BOOST_AUTO_TEST_CASE(TestGetGuidFromString)
     char buffer[] = "1da1ef8f-39ff-4d62-8b72-e8e9f3371801";
     SqlLen reslen = sizeof(buffer) - 1;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer) - 1, &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer) - 1, &reslen);
 
     ignite::Guid guid = appBuf.GetGuid();
 
@@ -130,9 +127,8 @@ BOOST_AUTO_TEST_CASE(TestPutBinaryToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     uint8_t binary[] = { 0x21, 0x84, 0xF4, 0xDC, 0x01, 0x00, 0xFF, 0xF0 };
 
@@ -146,9 +142,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     std::string testString("Test string");
 
@@ -162,9 +157,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToWstring)
 {
     wchar_t buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_WCHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_WCHAR, buffer, sizeof(buffer), &reslen);
 
     std::string testString("Test string");
 
@@ -176,9 +170,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToLong)
 {
     long numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutString("424242424");
     BOOST_CHECK(numBuf == 424242424L);
@@ -191,9 +184,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToTiny)
 {
     int8_t numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_TINYINT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_TINYINT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutString("12");
     BOOST_CHECK(numBuf == 12);
@@ -206,9 +198,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToFloat)
 {
     float numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutString("12.21");
     BOOST_CHECK_CLOSE_FRACTION(numBuf, 12.21, FLOAT_PRECISION);
@@ -221,9 +212,8 @@ BOOST_AUTO_TEST_CASE(TestPutIntToFloat)
 {
     float numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutInt8(5);
     BOOST_CHECK_CLOSE_FRACTION(numBuf, 5.0, FLOAT_PRECISION);
@@ -248,9 +238,8 @@ BOOST_AUTO_TEST_CASE(TestPutFloatToShort)
 {
     short numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutDouble(5.42);
     BOOST_CHECK(numBuf == 5);
@@ -270,7 +259,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToDouble)
     double numBuf;
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, sizeof(numBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, sizeof(numBuf), &reslen);
 
     common::Decimal decimal;
 
@@ -299,7 +288,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToLong)
     long numBuf;
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen);
 
     common::Decimal decimal;
 
@@ -326,7 +315,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToString)
     char strBuf[64];
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     common::Decimal decimal;
 
@@ -353,7 +342,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToNumeric)
     SQL_NUMERIC_STRUCT buf;
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_NUMERIC, &buf, sizeof(buf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_NUMERIC, &buf, sizeof(buf), &reslen);
 
     common::Decimal decimal;
 
@@ -404,7 +393,7 @@ BOOST_AUTO_TEST_CASE(TestPutDateToString)
     char strBuf[64] = { 0 };
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     Date date = BinaryUtils::MakeDateGmt(1999, 2, 22);
 
@@ -418,7 +407,7 @@ BOOST_AUTO_TEST_CASE(TestPutTimestampToString)
     char strBuf[64] = { 0 };
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     Timestamp date = BinaryUtils::MakeTimestampGmt(2018, 11, 1, 17, 45, 59);
 
@@ -432,10 +421,7 @@ BOOST_AUTO_TEST_CASE(TestPutDateToDate)
     SQL_DATE_STRUCT buf = { 0 };
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Date date = BinaryUtils::MakeDateGmt(1984, 5, 27);
 
@@ -451,10 +437,7 @@ BOOST_AUTO_TEST_CASE(TestPutTimestampToDate)
     SQL_DATE_STRUCT buf = { 0 };
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = BinaryUtils::MakeTimestampGmt(2004, 8, 14, 6, 34, 51, 573948623);
 
@@ -470,10 +453,7 @@ BOOST_AUTO_TEST_CASE(TestPutTimestampToTimestamp)
     SQL_TIMESTAMP_STRUCT buf = { 0 };
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = BinaryUtils::MakeTimestampGmt(2004, 8, 14, 6, 34, 51, 573948623);
 
@@ -494,10 +474,7 @@ BOOST_AUTO_TEST_CASE(TestPutDateToTimestamp)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Date date = BinaryUtils::MakeDateGmt(1984, 5, 27);
 
@@ -516,9 +493,8 @@ BOOST_AUTO_TEST_CASE(TestGetStringFromLong)
 {
     long numBuf = 42;
     SqlLen reslen = sizeof(numBuf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, reslen, &reslen);
 
     std::string res = appBuf.GetString(32);
 
@@ -535,9 +511,8 @@ BOOST_AUTO_TEST_CASE(TestGetStringFromDouble)
 {
     double numBuf = 43.36;
     SqlLen reslen = sizeof(numBuf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, reslen, &reslen);
 
     std::string res = appBuf.GetString(32);
 
@@ -554,9 +529,8 @@ BOOST_AUTO_TEST_CASE(TestGetStringFromString)
 {
     char buf[] = "Some data 32d2d5hs";
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen);
 
     std::string res = appBuf.GetString(reslen);
 
@@ -567,9 +541,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromUshort)
 {
     unsigned short numBuf = 7162;
     SqlLen reslen = sizeof(numBuf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT, &numBuf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT, &numBuf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -584,9 +557,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromString)
 {
     char buf[] = "28.562";
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -601,9 +573,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromFloat)
 {
     float buf = 207.49f;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -618,9 +589,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromDouble)
 {
     double buf = 893.162;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -635,9 +605,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromString)
 {
     char buf[] = "39";
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -660,9 +629,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromFloat)
 {
     float buf = -107.49f;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -685,9 +653,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromDouble)
 {
     double buf = 42.97f;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -710,9 +677,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromBigint)
 {
     uint64_t buf = 19;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -744,22 +710,19 @@ BOOST_AUTO_TEST_CASE(TestGetIntWithOffset)
         { 42, sizeof(uint64_t) }
     };
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen);
 
     int64_t val = appBuf.GetInt64();
 
     BOOST_CHECK(val == 12);
 
-    offset += sizeof(TestStruct);
+    appBuf.SetByteOffset(sizeof(TestStruct));
 
     val = appBuf.GetInt64();
 
     BOOST_CHECK(val == 42);
 
-    offsetPtr = 0;
+    appBuf.SetByteOffset(0);
 
     val = appBuf.GetInt64();
 
@@ -779,10 +742,7 @@ BOOST_AUTO_TEST_CASE(TestSetStringWithOffset)
         { "", 0 }
     };
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen);
 
     appBuf.PutString("Hello Ignite!");
 
@@ -792,7 +752,7 @@ BOOST_AUTO_TEST_CASE(TestSetStringWithOffset)
     BOOST_CHECK(res == "Hello Ignite!");
     BOOST_CHECK(res.size() == strlen("Hello Ignite!"));
 
-    offset += sizeof(TestStruct);
+    appBuf.SetByteOffset(sizeof(TestStruct));
 
     appBuf.PutString("Hello with offset!");
 
@@ -814,10 +774,7 @@ BOOST_AUTO_TEST_CASE(TestGetDateFromString)
     char buf[] = "1999-02-22";
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen);
 
     Date date = appBuf.GetDate();
 
@@ -840,10 +797,7 @@ BOOST_AUTO_TEST_CASE(TestGetTimestampFromString)
     char buf[] = "2018-11-01 17:45:59";
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen);
 
     Timestamp date = appBuf.GetTimestamp();
 
@@ -871,10 +825,7 @@ BOOST_AUTO_TEST_CASE(TestGetDateFromDate)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Date date = appBuf.GetDate();
 
@@ -902,10 +853,7 @@ BOOST_AUTO_TEST_CASE(TestGetTimestampFromDate)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = appBuf.GetTimestamp();
 
@@ -937,10 +885,7 @@ BOOST_AUTO_TEST_CASE(TestGetTimestampFromTimestamp)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = appBuf.GetTimestamp();
 
@@ -973,10 +918,7 @@ BOOST_AUTO_TEST_CASE(TestGetDateFromTimestamp)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Date date = appBuf.GetDate();
 
diff --git a/modules/platforms/cpp/odbc-test/src/column_test.cpp b/modules/platforms/cpp/odbc-test/src/column_test.cpp
index 6cbea8b648f4b..a5b0d0854103a 100644
--- a/modules/platforms/cpp/odbc-test/src/column_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/column_test.cpp
@@ -66,9 +66,8 @@ BOOST_AUTO_TEST_CASE(TestColumnShort)
 
     short shortBuf = 0;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &shortBuf, sizeof(shortBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &shortBuf, sizeof(shortBuf), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -114,9 +113,8 @@ BOOST_AUTO_TEST_CASE(TestColumnString)
 
     char strBuf[1024] = {};
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -164,9 +162,8 @@ BOOST_AUTO_TEST_CASE(TestColumnStringSeveral)
 
     std::string strBuf(data.size() / 3 + 2, 0);
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf[0], strBuf.size(), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf[0], strBuf.size(), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -246,9 +243,8 @@ BOOST_AUTO_TEST_CASE(TestColumnMultiString)
 
     char strBuf[1024] = {};
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     BOOST_REQUIRE(column1.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 3a3858802459e..9aca77de784ed 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -33,6 +33,7 @@
 #include <boost/test/unit_test.hpp>
 
 #include "ignite/ignite.h"
+#include "ignite/common/fixed_size_array.h"
 #include "ignite/ignition.h"
 #include "ignite/impl/binary/binary_utils.h"
 
@@ -324,10 +325,10 @@ struct QueriesTestSuiteFixture
     }
 
     /**
-     * Insert requested number of TestType vlaues with all defaults except
+     * Insert requested number of TestType values with all defaults except
      * for the strFields, which are generated using getTestString().
      *
-     * @param num Number of records to insert.
+     * @param recordsNum Number of records to insert.
      * @param merge Set to true to use merge instead.
      */
     void InsertTestStrings(int recordsNum, bool merge = false)
@@ -385,6 +386,317 @@ struct QueriesTestSuiteFixture
             BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
     }
 
+    /**
+     * Insert requested number of TestType values in a batch.
+     *
+     * @param from Index to start from.
+     * @param to Index to stop.
+     * @param expectedToAffect Expected number of affected records.
+     * @param merge Set to true to use merge instead of insert.
+     * @return Records inserted.
+     */
+    int InsertTestBatch(int from, int to, int expectedToAffect, bool merge = false)
+    {
+        SQLCHAR insertReq[] = "INSERT "
+            "INTO TestType(_key, i8Field, i16Field, i32Field, strField, floatField, doubleField, boolField, dateField, "
+            "timestampField) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+        SQLCHAR mergeReq[] = "MERGE "
+            "INTO TestType(_key, i8Field, i16Field, i32Field, strField, floatField, doubleField, boolField, dateField, "
+            "timestampField) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+        SQLRETURN ret;
+
+        int recordsNum = to - from;
+
+        ret = SQLPrepare(stmt, merge ? mergeReq : insertReq, SQL_NTS);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        FixedSizeArray<int64_t> keys(recordsNum);
+        FixedSizeArray<int8_t> i8Fields(recordsNum);
+        FixedSizeArray<int16_t> i16Fields(recordsNum);
+        FixedSizeArray<int32_t> i32Fields(recordsNum);
+        FixedSizeArray<char> strFields(recordsNum * 1024);
+        FixedSizeArray<float> floatFields(recordsNum);
+        FixedSizeArray<double> doubleFields(recordsNum);
+        FixedSizeArray<bool> boolFields(recordsNum);
+        FixedSizeArray<SQL_DATE_STRUCT> dateFields(recordsNum);
+        FixedSizeArray<SQL_TIME_STRUCT> timeFields(recordsNum);
+        FixedSizeArray<SQL_TIMESTAMP_STRUCT> timestampFields(recordsNum);
+
+        FixedSizeArray<SQLLEN> strFieldsLen(recordsNum);
+
+        BOOST_CHECKPOINT("Filling param data");
+
+        for (int i = 0; i < recordsNum; ++i)
+        {
+            int seed = from + i;
+
+            keys[i] = seed;
+            i8Fields[i] = seed * 8;
+            i16Fields[i] = seed * 16;
+            i32Fields[i] = seed * 32;
+
+            std::string val = getTestString(seed);
+            strncpy(strFields.GetData() + 1024 * i, val.c_str(), 1023);
+            strFieldsLen[i] = val.size();
+
+            floatFields[i] = seed * 0.5f;
+            doubleFields[i] = seed * 0.25f;
+            boolFields[i] = seed % 2 == 0;
+
+            dateFields[i].year = 2017 + seed / 365;
+            dateFields[i].month = ((seed / 28) % 12) + 1;
+            dateFields[i].day = (seed % 28) + 1;
+
+            timeFields[i].hour = (seed / 3600) % 24;
+            timeFields[i].minute = (seed / 60) % 60;
+            timeFields[i].second = seed % 60;
+
+            timestampFields[i].year = dateFields[i].year;
+            timestampFields[i].month = dateFields[i].month;
+            timestampFields[i].day = dateFields[i].day;
+            timestampFields[i].hour = timeFields[i].hour;
+            timestampFields[i].minute = timeFields[i].minute;
+            timestampFields[i].second = timeFields[i].second;
+            timestampFields[i].fraction = std::abs(seed * 914873) % 1000000000;
+        }
+
+        SQLULEN setsProcessed = 0;
+
+        BOOST_CHECKPOINT("Setting processed pointer");
+        ret = SQLSetStmtAttr(stmt, SQL_ATTR_PARAMS_PROCESSED_PTR, &setsProcessed, SQL_IS_POINTER);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding keys");
+        ret = SQLBindParameter(stmt, 1, SQL_PARAM_INPUT, SQL_C_SBIGINT, SQL_BIGINT, 0, 0, keys.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding i8Fields");
+        ret = SQLBindParameter(stmt, 2, SQL_PARAM_INPUT, SQL_C_STINYINT, SQL_TINYINT, 0, 0, i8Fields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding i16Fields");
+        ret = SQLBindParameter(stmt, 3, SQL_PARAM_INPUT, SQL_C_SSHORT, SQL_SMALLINT, 0, 0, i16Fields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding i32Fields");
+        ret = SQLBindParameter(stmt, 4, SQL_PARAM_INPUT, SQL_C_SLONG, SQL_INTEGER, 0, 0, i32Fields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding strFields");
+        ret = SQLBindParameter(stmt, 5, SQL_PARAM_INPUT, SQL_C_CHAR, SQL_VARCHAR, 1024, 0, strFields.GetData(), 1024, strFieldsLen.GetData());
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding floatFields");
+        ret = SQLBindParameter(stmt, 6, SQL_PARAM_INPUT, SQL_C_FLOAT, SQL_FLOAT, 0, 0, floatFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding doubleFields");
+        ret = SQLBindParameter(stmt, 7, SQL_PARAM_INPUT, SQL_C_DOUBLE, SQL_DOUBLE, 0, 0, doubleFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding boolFields");
+        ret = SQLBindParameter(stmt, 8, SQL_PARAM_INPUT, SQL_C_BIT, SQL_BIT, 0, 0, boolFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding dateFields");
+        ret = SQLBindParameter(stmt, 9, SQL_PARAM_INPUT, SQL_C_DATE, SQL_DATE, 0, 0, dateFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding timestampFields");
+        ret = SQLBindParameter(stmt, 10, SQL_PARAM_INPUT, SQL_C_TIMESTAMP, SQL_TIMESTAMP, 0, 0, timestampFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Setting paramset size");
+        ret = SQLSetStmtAttr(stmt, SQL_ATTR_PARAMSET_SIZE, reinterpret_cast<SQLPOINTER>(recordsNum), 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Executing query");
+        ret = SQLExecute(stmt);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        SQLLEN affected = 0;
+        ret = SQLRowCount(stmt, &affected);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECK_EQUAL(affected, expectedToAffect);
+
+        BOOST_CHECKPOINT("Getting next result set");
+        ret = SQLMoreResults(stmt);
+
+        if (ret != SQL_NO_DATA)
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Resetting parameters.");
+        ret = SQLFreeStmt(stmt, SQL_RESET_PARAMS);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Setting paramset size");
+        ret = SQLSetStmtAttr(stmt, SQL_ATTR_PARAMSET_SIZE, reinterpret_cast<SQLPOINTER>(1), 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        return static_cast<int>(setsProcessed);
+    }
+
+    void InsertBatchSelect(int recordsNum)
+    {
+        Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
+
+        // Inserting values.
+        int inserted = InsertTestBatch(0, recordsNum, recordsNum);
+
+        BOOST_REQUIRE_EQUAL(inserted, recordsNum);
+
+        int64_t key = 0;
+        char strField[1024] = { 0 };
+        SQLLEN strFieldLen = 0;
+
+        // Binding columns.
+        SQLRETURN ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Binding columns.
+        ret = SQLBindCol(stmt, 2, SQL_C_CHAR, &strField, sizeof(strField), &strFieldLen);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Just selecting everything to make sure everything is OK
+        SQLCHAR selectReq[] = "SELECT _key, strField FROM TestType ORDER BY _key";
+
+        ret = SQLExecDirect(stmt, selectReq, sizeof(selectReq));
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        int selectedRecordsNum = 0;
+
+        ret = SQL_SUCCESS;
+
+        while (ret == SQL_SUCCESS)
+        {
+            ret = SQLFetch(stmt);
+
+            if (ret == SQL_NO_DATA)
+                break;
+
+            if (!SQL_SUCCEEDED(ret))
+                BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+            std::string expectedStr = getTestString(selectedRecordsNum);
+            int64_t expectedKey = selectedRecordsNum;
+
+            BOOST_CHECK_EQUAL(key, expectedKey);
+
+            BOOST_CHECK_EQUAL(std::string(strField, strFieldLen), expectedStr);
+
+            ++selectedRecordsNum;
+        }
+
+        BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
+    }
+
+    void InsertNonFullBatchSelect(int recordsNum, int splitAt)
+    {
+        Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
+
+        // Inserting values.
+        int inserted = InsertTestBatch(splitAt, recordsNum, recordsNum - splitAt);
+
+        BOOST_REQUIRE_EQUAL(inserted, recordsNum - splitAt);
+
+        inserted = InsertTestBatch(0, recordsNum, splitAt);
+
+        BOOST_REQUIRE_EQUAL(inserted, splitAt);
+
+        int64_t key = 0;
+        char strField[1024] = { 0 };
+        SQLLEN strFieldLen = 0;
+
+        // Binding columns.
+        SQLRETURN ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Binding columns.
+        ret = SQLBindCol(stmt, 2, SQL_C_CHAR, &strField, sizeof(strField), &strFieldLen);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Just selecting everything to make sure everything is OK
+        SQLCHAR selectReq[] = "SELECT _key, strField FROM TestType ORDER BY _key";
+
+        ret = SQLExecDirect(stmt, selectReq, sizeof(selectReq));
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        int selectedRecordsNum = 0;
+
+        ret = SQL_SUCCESS;
+
+        while (ret == SQL_SUCCESS)
+        {
+            ret = SQLFetch(stmt);
+
+            if (ret == SQL_NO_DATA)
+                break;
+
+            if (!SQL_SUCCEEDED(ret))
+                BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+            std::string expectedStr = getTestString(selectedRecordsNum);
+            int64_t expectedKey = selectedRecordsNum;
+
+            BOOST_CHECK_EQUAL(key, expectedKey);
+
+            BOOST_CHECK_EQUAL(std::string(strField, strFieldLen), expectedStr);
+
+            ++selectedRecordsNum;
+        }
+
+        BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
+    }
+
     /** Node started during the test. */
     Ignite grid;
 
@@ -1361,6 +1673,76 @@ BOOST_AUTO_TEST_CASE(TestInsertMergeSelect)
     BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
 }
 
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2)
+{
+    InsertBatchSelect(2);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect100)
+{
+    InsertBatchSelect(100);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1000)
+{
+    InsertBatchSelect(1000);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1023)
+{
+    InsertBatchSelect(1024);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1024)
+{
+    InsertBatchSelect(1024);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1025)
+{
+    InsertBatchSelect(1025);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2000)
+{
+    InsertBatchSelect(2000);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2047)
+{
+    InsertBatchSelect(2048);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2048)
+{
+    InsertBatchSelect(2048);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2049)
+{
+    InsertBatchSelect(2049);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect900)
+{
+    InsertNonFullBatchSelect(900, 42);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect1500)
+{
+    InsertNonFullBatchSelect(1500, 100);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect4500)
+{
+    InsertNonFullBatchSelect(4500, 1500);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect4096)
+{
+    InsertNonFullBatchSelect(4096, 1024);
+}
+
 BOOST_AUTO_TEST_CASE(TestParamsNum)
 {
     Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
diff --git a/modules/platforms/cpp/odbc-test/src/row_test.cpp b/modules/platforms/cpp/odbc-test/src/row_test.cpp
index f38e9c5b58f1d..1ae34c12feadf 100644
--- a/modules/platforms/cpp/odbc-test/src/row_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/row_test.cpp
@@ -82,12 +82,11 @@ void CheckRowData(Row& row, size_t rowIdx)
     char strBuf[1024];
     SQLGUID guidBuf;
     char bitBuf;
-    int* offset = 0;
 
-    ApplicationDataBuffer appLongBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_LONG, &longBuf, sizeof(longBuf), &reslen, &offset);
-    ApplicationDataBuffer appStrBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, &offset);
-    ApplicationDataBuffer appGuidBuf(type_traits::IGNITE_ODBC_C_TYPE_GUID, &guidBuf, sizeof(guidBuf), &reslen, &offset);
-    ApplicationDataBuffer appBitBuf(type_traits::IGNITE_ODBC_C_TYPE_BIT, &bitBuf, sizeof(bitBuf), &reslen, &offset);
+    ApplicationDataBuffer appLongBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_LONG, &longBuf, sizeof(longBuf), &reslen);
+    ApplicationDataBuffer appStrBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
+    ApplicationDataBuffer appGuidBuf(type_traits::IGNITE_ODBC_C_TYPE_GUID, &guidBuf, sizeof(guidBuf), &reslen);
+    ApplicationDataBuffer appBitBuf(type_traits::IGNITE_ODBC_C_TYPE_BIT, &bitBuf, sizeof(bitBuf), &reslen);
 
     // Checking size.
     BOOST_REQUIRE(row.GetSize() == 4);
diff --git a/modules/platforms/cpp/odbc/Makefile.am b/modules/platforms/cpp/odbc/Makefile.am
index b0cc5f87133fc..c2fb42e19edff 100644
--- a/modules/platforms/cpp/odbc/Makefile.am
+++ b/modules/platforms/cpp/odbc/Makefile.am
@@ -52,6 +52,7 @@ libignite_odbc_la_SOURCES = \
     os/linux/src/system/socket_client.cpp \
     src/app/application_data_buffer.cpp \
     src/app/parameter.cpp \
+    src/app/parameter_set.cpp \
     src/common_types.cpp \
     src/config/configuration.cpp \
     src/config/connection_info.cpp \
@@ -68,6 +69,7 @@ libignite_odbc_la_SOURCES = \
     src/dsn_config.cpp \
     src/query/column_metadata_query.cpp \
     src/query/data_query.cpp \
+    src/query/batch_query.cpp \
     src/query/foreign_keys_query.cpp \
     src/query/primary_keys_query.cpp \
     src/query/table_metadata_query.cpp \
@@ -76,6 +78,7 @@ libignite_odbc_la_SOURCES = \
     src/protocol_version.cpp \
     src/result_page.cpp \
     src/row.cpp \
+    src/message.cpp \
     src/column.cpp \
     src/statement.cpp \
     src/type_traits.cpp \
diff --git a/modules/platforms/cpp/odbc/include/Makefile.am b/modules/platforms/cpp/odbc/include/Makefile.am
index 073dcaa1898ba..1408aee3cb52b 100644
--- a/modules/platforms/cpp/odbc/include/Makefile.am
+++ b/modules/platforms/cpp/odbc/include/Makefile.am
@@ -22,6 +22,7 @@ noinst_HEADERS = \
     ignite/odbc/query/table_metadata_query.h \
     ignite/odbc/query/special_columns_query.h \
     ignite/odbc/query/type_info_query.h \
+    ignite/odbc/query/batch_query.h \
     ignite/odbc/query/data_query.h \
     ignite/odbc/query/foreign_keys_query.h \
     ignite/odbc/query/column_metadata_query.h \
@@ -35,6 +36,7 @@ noinst_HEADERS = \
     ignite/odbc/parser.h \
     ignite/odbc/app/application_data_buffer.h \
     ignite/odbc/app/parameter.h \
+    ignite/odbc/app/parameter_set.h \
     ignite/odbc/row.h \
     ignite/odbc/utility.h \
     ignite/odbc/environment.h \
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h b/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h
index 18ac36aead536..b7989c55982fa 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h
@@ -54,10 +54,9 @@ namespace ignite
                  * @param buffer Data buffer pointer.
                  * @param buflen Data buffer length.
                  * @param reslen Resulting data length.
-                 * @param offset Pointer to buffer and reslen offset pointer.
                  */
                 ApplicationDataBuffer(type_traits::IgniteSqlType type, void* buffer,
-                    SqlLen buflen, SqlLen* reslen, int** offset = 0);
+                    SqlLen buflen, SqlLen* reslen);
 
                 /**
                  * Copy constructor.
@@ -80,13 +79,23 @@ namespace ignite
                 ApplicationDataBuffer& operator=(const ApplicationDataBuffer& other);
 
                 /**
-                 * Set pointer to offset pointer.
+                 * Set offset in bytes for all bound pointers.
                  *
-                 * @param offset Pointer to offset pointer.
+                 * @param offset Offset.
                  */
-                void SetPtrToOffsetPtr(int** offset)
+                void SetByteOffset(int offset)
                 {
-                    this->offset = offset;
+                    this->byteOffset = offset;
+                }
+
+                /**
+                 * Set offset in elements for all bound pointers.
+                 *
+                 * @param
+                 */
+                void SetElementOffset(SqlUlen idx)
+                {
+                    this->elementOffset = idx;
                 }
 
                 /**
@@ -313,6 +322,13 @@ namespace ignite
                  */
                 SqlLen GetDataAtExecSize() const;
 
+                /**
+                 * Get single element size.
+                 *
+                 * @return Size of the single element.
+                 */
+                SqlLen GetElementSize() const;
+
                 /**
                  * Get size of the input buffer.
                  *
@@ -392,10 +408,11 @@ namespace ignite
                  * Apply buffer offset to pointer.
                  * Adds offset to pointer if offset pointer is not null.
                  * @param ptr Pointer.
+                 * @param elemSize Element size.
                  * @return Pointer with applied offset.
                  */
                 template<typename T>
-                T* ApplyOffset(T* ptr) const;
+                T* ApplyOffset(T* ptr, size_t elemSize) const;
 
                 /** Underlying data type. */
                 type_traits::IgniteSqlType type;
@@ -409,8 +426,11 @@ namespace ignite
                 /** Result length. */
                 SqlLen* reslen;
 
-                /** Pointer to implementation pointer to application offset */
-                int** offset;
+                /** Current byte offset */
+                int byteOffset;
+
+                /** Current element offset. */
+                SqlUlen elementOffset;
             };
 
             /** Column binging map type alias. */
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h
index 1cf85b514aa59..cdaaead40ee3d 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h
@@ -22,12 +22,10 @@
 
 #include <map>
 
-#include <ignite/guid.h>
 #include <ignite/impl/binary/binary_writer_impl.h>
 #include <ignite/impl/binary/binary_reader_impl.h>
 
 #include "ignite/odbc/app/application_data_buffer.h"
-#include "ignite/odbc/type_traits.h"
 
 namespace ignite
 {
@@ -70,7 +68,7 @@ namespace ignite
                 ~Parameter();
 
                 /**
-                 * Copy assigment operator.
+                 * Assignment operator.
                  *
                  * @param other Other instance.
                  * @return This.
@@ -78,10 +76,12 @@ namespace ignite
                 Parameter& operator=(const Parameter& other);
 
                 /**
-                 * Write request using provided writer.
+                 * Write parameter using provided writer.
                  * @param writer Writer.
+                 * @param offset Offset for the buffer.
+                 * @param idx Index for the array-of-parameters case.
                  */
-                void Write(impl::binary::BinaryWriterImpl& writer) const;
+                void Write(impl::binary::BinaryWriterImpl& writer, int offset = 0, SqlUlen idx = 0) const;
 
                 /**
                  * Get data buffer.
@@ -90,6 +90,13 @@ namespace ignite
                  */
                 ApplicationDataBuffer& GetBuffer();
 
+                /**
+                 * Get data buffer.
+                 *
+                 * @return underlying ApplicationDataBuffer instance.
+                 */
+                const ApplicationDataBuffer& GetBuffer() const;
+
                 /**
                  * Reset stored at-execution data.
                  */
@@ -128,9 +135,6 @@ namespace ignite
                 /** Stored at-execution data. */
                 std::vector<int8_t> storedData;
             };
-
-            /** Parameter binging map type alias. */
-            typedef std::map<uint16_t, Parameter> ParameterBindingMap;
         }
     }
 }
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
new file mode 100644
index 0000000000000..2ab55808c073c
--- /dev/null
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _IGNITE_ODBC_APP_PARAMETER_SET
+#define _IGNITE_ODBC_APP_PARAMETER_SET
+
+#include <stdint.h>
+
+#include <map>
+
+#include <ignite/impl/binary/binary_writer_impl.h>
+#include <ignite/impl/binary/binary_reader_impl.h>
+
+#include "ignite/odbc/app/parameter.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        namespace app
+        {
+            /**
+             * Parameter set.
+             */
+            class ParameterSet
+            {
+                /** Parameter binging map type alias. */
+                typedef std::map<uint16_t, Parameter> ParameterBindingMap;
+
+                /** Parameter meta vector. */
+                typedef std::vector<int8_t> ParameterTypeVector;
+            public:
+                /**
+                 * Default constructor.
+                 */
+                ParameterSet();
+
+                /**
+                 * Destructor.
+                 */
+                ~ParameterSet()
+                {
+                    // No-op.
+                }
+
+                /**
+                 * Set parameters set size.
+                 *
+                 * @param size Size of the parameter set.
+                 */
+                void SetParamSetSize(SqlUlen size);
+
+                /**
+                 * Bind parameter.
+                 * 
+                 * @param paramIdx Parameter index.
+                 * @param param Parameter.
+                 */
+                void BindParameter(uint16_t paramIdx, const Parameter& param);
+
+                /**
+                 * Unbind specified parameter.
+                 *
+                 * @param paramIdx Parameter index.
+                 */
+                void UnbindParameter(uint16_t paramIdx);
+
+                /**
+                 * Unbind all parameters.
+                 */
+                void UnbindAll();
+
+                /**
+                 * Get number of binded parameters.
+                 *
+                 * @return Number of binded parameters.
+                 */
+                uint16_t GetParametersNumber() const;
+
+                /**
+                 * Set parameter binding offset pointer.
+                 *
+                 * @param ptr Parameter binding offset pointer.
+                 */
+                void SetParamBindOffsetPtr(int* ptr);
+
+                /**
+                 * Get parameter binding offset pointer.
+                 *
+                 * @return Parameter binding offset pointer.
+                 */
+                int* GetParamBindOffsetPtr();
+
+                /**
+                 * Prepare parameters set for statement execution.
+                 */
+                void Prepare();
+
+                /**
+                 * Check if the data at-execution is needed.
+                 *
+                 * @return True if the data at execution is needed.
+                 */
+                bool IsDataAtExecNeeded() const;
+
+                /**
+                 * Update parameter types metadata.
+                 *
+                 * @param meta Types metadata.
+                 */
+                void UpdateParamsTypes(const ParameterTypeVector& meta);
+
+                /**
+                 * Get type id of the parameter.
+                 *
+                 * @param idx Parameter index.
+                 * @param dflt Default value to return if the type can not be found.
+                 * @return Type ID of the parameter or dflt, if the type can not be returned.
+                 */
+                int8_t GetParamType(int16_t idx, int8_t dflt);
+
+                /**
+                 * Get expected parameters number.
+                 * Using metadata. If metadata was not updated returns zero.
+                 *
+                 * @return Expected parameters number.
+                 */
+                uint16_t GetExpectedParamNum();
+
+                /**
+                 * Check if the metadata was set for the parameter set.
+                 * 
+                 * @return True if the metadata was set for the parameter set. 
+                 */
+                bool IsMetadataSet() const;
+
+                /**
+                 * Check if the parameter selected for putting data at-execution.
+                 *
+                 * @return True if the parameter selected for putting data at-execution.
+                 */
+                bool IsParameterSelected() const;
+
+                /**
+                 * Get parameter by index.
+                 *
+                 * @param idx Index.
+                 * @return Parameter or null, if parameter is not bound.
+                 */
+                Parameter* GetParameter(uint16_t idx);
+
+                /**
+                 * Get selected parameter.
+                 *
+                 * @return Parameter or null, if parameter is not bound.
+                 */
+                Parameter* GetSelectedParameter();
+
+                /**
+                 * Internally selects next parameter for putting data at-execution.
+                 *
+                 * @return Parameter if found and null otherwise.
+                 */
+                Parameter* SelectNextParameter();
+
+                /**
+                 * Write only first row of the param set using provided writer.
+                 * @param writer Writer.
+                 */
+                void Write(impl::binary::BinaryWriterImpl& writer) const;
+
+                /**
+                 * Write rows of the param set in interval [begin, end) using provided writer.
+                 * @param writer Writer.
+                 * @param begin Beginng of the interval.
+                 * @param end End of the interval.
+                 * @param last Last page flag.
+                 */
+                void Write(impl::binary::BinaryWriterImpl& writer, SqlUlen begin, SqlUlen end, bool last) const;
+
+                /**
+                 * Calculate row length.
+                 *
+                 * @return Row length.
+                 */
+                int32_t CalculateRowLen() const;
+
+                /**
+                 * Get parameter set size.
+                 *
+                 * @return Number of rows in set.
+                 */
+                int32_t GetParamSetSize() const;
+
+                /**
+                 * Set number of parameters processed in batch.
+                 *
+                 * @param processed Processed.
+                 */
+                void SetParamsProcessed(SqlUlen processed) const;
+
+                /**
+                 * Number of processed params should be written using provided address.
+                 *
+                 * @param ptr Pointer.
+                 */
+                void SetParamsProcessedPtr(SqlUlen* ptr);
+
+                /**
+                 * Get pointer to write number of parameters processed in batch.
+                 *
+                 * @return Pointer to write number of parameters processed in batch.
+                 */
+                SqlUlen* GetParamsProcessedPtr();
+
+            private:
+                /**
+                 * Write single row of the param set using provided writer.
+                 * @param writer Writer.
+                 * @param idx Row index.
+                 */
+                void WriteRow(impl::binary::BinaryWriterImpl& writer, SqlUlen idx) const;
+
+                IGNITE_NO_COPY_ASSIGNMENT(ParameterSet);
+
+                /** Parameters. */
+                ParameterBindingMap parameters;
+
+                /** Parameter types. */
+                ParameterTypeVector paramTypes;
+
+                /** Offset added to pointers to change binding of parameters. */
+                int* paramBindOffset;
+
+                /** Processed parameters. */
+                SqlUlen* processedParamRows;
+
+                /** Parameter set size. */
+                SqlUlen paramSetSize;
+
+                /** Current position in parametor set. */
+                SqlUlen paramSetPos;
+
+                /** Index of the parameter, which is currently being set. */
+                uint16_t currentParamIdx;
+
+                /** Parameter types are set. */
+                bool typesSet;
+            };
+        }
+    }
+}
+
+#endif //_IGNITE_ODBC_APP_PARAMETER_SET
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h b/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
index 517fe4e0be17d..408816a6d8f54 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
@@ -132,7 +132,13 @@ namespace ignite
              * The connection timeout period expired before the data source
              * responded to the request.
              */
-            SQL_STATE_HYT01_CONNECTIOIN_TIMEOUT
+            SQL_STATE_HYT01_CONNECTIOIN_TIMEOUT,
+
+            /** Invalid buffer type. */
+            SQL_STATE_HY003_INVALID_APPLICATION_BUFFER_TYPE,
+
+            /** Invalid parameter type. */
+            SQL_STATE_HY105_INVALID_PARAMETER_TYPE
         };
 
         /**
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h b/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
index 6937fcc6f3196..95e4f548c1f9c 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
@@ -45,14 +45,14 @@ namespace ignite
                  *
                  * @return Diagnostic record.
                  */
-                virtual const diagnostic::DiagnosticRecordStorage& GetDiagnosticRecords() const = 0;
+                virtual const DiagnosticRecordStorage& GetDiagnosticRecords() const = 0;
 
                 /**
                  * Get diagnostic record.
                  *
                  * @return Diagnostic record.
                  */
-                virtual diagnostic::DiagnosticRecordStorage& GetDiagnosticRecords() = 0;
+                virtual DiagnosticRecordStorage& GetDiagnosticRecords() = 0;
 
                 /**
                  * Add new status record.
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
index a2bbd99adce3c..cecdac67f9d36 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
@@ -24,11 +24,11 @@
 #include "ignite/impl/binary/binary_writer_impl.h"
 #include "ignite/impl/binary/binary_reader_impl.h"
 
-#include "ignite/odbc/utility.h"
 #include "ignite/odbc/result_page.h"
+#include "ignite/odbc/protocol_version.h"
 #include "ignite/odbc/meta/column_meta.h"
 #include "ignite/odbc/meta/table_meta.h"
-#include "ignite/odbc/app/parameter.h"
+#include "ignite/odbc/app/parameter_set.h"
 
 namespace ignite
 {
@@ -48,7 +48,9 @@ namespace ignite
 
             REQUEST_TYPE_GET_TABLES_METADATA = 6,
 
-            REQUEST_TYPE_GET_PARAMS_METADATA = 7
+            REQUEST_TYPE_GET_PARAMS_METADATA = 7,
+
+            REQUEST_TYPE_EXECUTE_SQL_QUERY_BATCH = 8
         };
 
         enum ResponseStatus
@@ -71,35 +73,18 @@ namespace ignite
              * @param distributedJoins Distributed joins flag.
              * @param enforceJoinOrder Enforce join order flag.
              */
-            HandshakeRequest(int64_t version, bool distributedJoins, bool enforceJoinOrder) :
-                version(version),
-                distributedJoins(distributedJoins),
-                enforceJoinOrder(enforceJoinOrder)
-            {
-                // No-op.
-            }
+            HandshakeRequest(int64_t version, bool distributedJoins, bool enforceJoinOrder);
 
             /**
              * Destructor.
              */
-            ~HandshakeRequest()
-            {
-                // No-op.
-            }
+            ~HandshakeRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_HANDSHAKE);
-
-                writer.WriteInt64(version);
-
-                writer.WriteBool(distributedJoins);
-                writer.WriteBool(enforceJoinOrder);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Protocol version. */
@@ -123,65 +108,80 @@ namespace ignite
              *
              * @param cache Cache name.
              * @param sql SQL query.
-             * @param argsNum Number of arguments.
+             * @param params Number of arguments.
              */
-            QueryExecuteRequest(const std::string& cache, const std::string& sql,
-                const app::ParameterBindingMap& params) :
-                cache(cache),
-                sql(sql),
-                params(params)
-            {
-                // No-op.
-            }
+            QueryExecuteRequest(const std::string& cache, const std::string& sql, const app::ParameterSet& params);
 
             /**
              * Destructor.
              */
-            ~QueryExecuteRequest()
-            {
-                // No-op.
-            }
+            ~QueryExecuteRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_EXECUTE_SQL_QUERY);
-                utility::WriteString(writer, cache);
-                utility::WriteString(writer, sql);
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
-                writer.WriteInt32(static_cast<int32_t>(params.size()));
+        private:
+            /** Cache name. */
+            std::string cache;
 
-                app::ParameterBindingMap::const_iterator i;
-                uint16_t prev = 0;
+            /** SQL query. */
+            std::string sql;
 
-                for (i = params.begin(); i != params.end(); ++i) {
-                    uint16_t current = i->first;
+            /** Parameters bindings. */
+            const app::ParameterSet& params;
+        };
 
-                    while ((current - prev) > 1) {
-                        writer.WriteNull();
-                        ++prev;
-                    }
+        /**
+         * Query execute batch request.
+         */
+        class QueryExecuteBatchtRequest
+        {
+        public:
+            /**
+             * Constructor.
+             *
+             * @param schema Schema.
+             * @param sql SQL query.
+             * @param params Query arguments.
+             * @param begin Beginng of the interval.
+             * @param end End of the interval.
+             */
+            QueryExecuteBatchtRequest(const std::string& schema, const std::string& sql,
+                                          const app::ParameterSet& params, SqlUlen begin, SqlUlen end, bool last);
 
-                    i->second.Write(writer);
+            /**
+             * Destructor.
+             */
+            ~QueryExecuteBatchtRequest();
 
-                    prev = current;
-                }
-            }
+            /**
+             * Write request using provided writer.
+             * @param writer Writer.
+             */
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
-            /** Cache name. */
-            std::string cache;
+            /** Schema name. */
+            std::string schema;
 
             /** SQL query. */
             std::string sql;
 
             /** Parameters bindings. */
-            const app::ParameterBindingMap& params;
-        };
+            const app::ParameterSet& params;
 
+            /** Beginng of the interval. */
+            SqlUlen begin;
+
+            /** End of the interval. */
+            SqlUlen end;
+
+            /** Last page flag. */
+            bool last;
+        };
 
         /**
          * Query close request.
@@ -194,28 +194,18 @@ namespace ignite
              *
              * @param queryId Query ID.
              */
-            QueryCloseRequest(int64_t queryId) : queryId(queryId)
-            {
-                // No-op.
-            }
+            QueryCloseRequest(int64_t queryId);
 
             /**
              * Destructor.
              */
-            ~QueryCloseRequest()
-            {
-                // No-op.
-            }
+            ~QueryCloseRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_CLOSE_SQL_QUERY);
-                writer.WriteInt64(queryId);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Query ID. */
@@ -234,31 +224,18 @@ namespace ignite
              * @param queryId Query ID.
              * @param pageSize Required page size.
              */
-            QueryFetchRequest(int64_t queryId, int32_t pageSize) :
-                queryId(queryId),
-                pageSize(pageSize)
-            {
-                // No-op.
-            }
+            QueryFetchRequest(int64_t queryId, int32_t pageSize);
 
             /**
              * Destructor.
              */
-            ~QueryFetchRequest()
-            {
-                // No-op.
-            }
+            ~QueryFetchRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_FETCH_SQL_QUERY);
-                writer.WriteInt64(queryId);
-                writer.WriteInt32(pageSize);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Query ID. */
@@ -281,34 +258,18 @@ namespace ignite
              * @param table Table name.
              * @param column Column name.
              */
-            QueryGetColumnsMetaRequest(const std::string& schema, const std::string& table, const std::string& column) :
-                schema(schema),
-                table(table),
-                column(column)
-            {
-                // No-op.
-            }
+            QueryGetColumnsMetaRequest(const std::string& schema, const std::string& table, const std::string& column);
 
             /**
              * Destructor.
              */
-            ~QueryGetColumnsMetaRequest()
-            {
-                // No-op.
-            }
+            ~QueryGetColumnsMetaRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_GET_COLUMNS_METADATA);
-                
-                utility::WriteString(writer, schema);
-                utility::WriteString(writer, table);
-                utility::WriteString(writer, column);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Schema search pattern. */
@@ -336,36 +297,18 @@ namespace ignite
              * @param tableTypes Table types search pattern.
              */
             QueryGetTablesMetaRequest(const std::string& catalog, const std::string& schema,
-                                      const std::string& table, const std::string& tableTypes) :
-                catalog(catalog),
-                schema(schema),
-                table(table),
-                tableTypes(tableTypes)
-            {
-                // No-op.
-            }
+                                      const std::string& table, const std::string& tableTypes);
 
             /**
              * Destructor.
              */
-            ~QueryGetTablesMetaRequest()
-            {
-                // No-op.
-            }
+            ~QueryGetTablesMetaRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_GET_TABLES_METADATA);
-
-                utility::WriteString(writer, catalog);
-                utility::WriteString(writer, schema);
-                utility::WriteString(writer, table);
-                utility::WriteString(writer, tableTypes);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Column search pattern. */
@@ -414,13 +357,7 @@ namespace ignite
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_GET_PARAMS_METADATA);
-
-                utility::WriteString(writer, cacheName);
-                utility::WriteString(writer, sqlQuery);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Cache name. */
@@ -439,33 +376,19 @@ namespace ignite
             /**
              * Constructor.
              */
-            Response() : status(RESPONSE_STATUS_FAILED), error()
-            {
-                // No-op.
-            }
+            Response();
 
             /**
              * Destructor.
              */
-            virtual ~Response()
-            {
-                // No-op.
-            }
+            virtual ~Response();
 
             /**
              * Read response using provided reader.
              * @param reader Reader.
              */
-            void Read(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                status = reader.ReadInt8();
+            void Read(impl::binary::BinaryReaderImpl& reader);
 
-                if (status == RESPONSE_STATUS_SUCCESS)
-                    ReadOnSuccess(reader);
-                else
-                    utility::ReadString(reader, error);;
-            }
-            
             /**
              * Get request processing status.
              * @return Status.
@@ -488,10 +411,7 @@ namespace ignite
             /**
              * Read data if response status is RESPONSE_STATUS_SUCCESS.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl&)
-            {
-                // No-op.
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl&);
 
         private:
             /** Request processing status. */
@@ -510,21 +430,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            HandshakeResponse() :
-                accepted(false),
-                protoVerSince(),
-                currentVer()
-            {
-                // No-op.
-            }
+            HandshakeResponse();
 
             /**
              * Destructor.
              */
-            ~HandshakeResponse()
-            {
-                // No-op.
-            }
+            ~HandshakeResponse();
 
             /**
              * Check if the handshake has been accepted.
@@ -558,16 +469,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                accepted = reader.ReadBool();
-
-                if (!accepted)
-                {
-                    utility::ReadString(reader, protoVerSince);
-                    utility::ReadString(reader, currentVer);
-                }
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Handshake accepted. */
             bool accepted;
@@ -588,18 +490,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryCloseResponse() : queryId(0)
-            {
-                // No-op.
-            }
+            QueryCloseResponse();
 
             /**
              * Destructor.
              */
-            ~QueryCloseResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryCloseResponse();
 
             /**
              * Get query ID.
@@ -615,10 +511,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                queryId = reader.ReadInt64();
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Query ID. */
             int64_t queryId;
@@ -633,18 +526,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryExecuteResponse() : queryId(0), meta()
-            {
-                // No-op.
-            }
+            QueryExecuteResponse();
 
             /**
              * Destructor.
              */
-            ~QueryExecuteResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryExecuteResponse();
 
             /**
              * Get query ID.
@@ -669,12 +556,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                queryId = reader.ReadInt64();
-
-                meta::ReadColumnMetaVector(reader, meta);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Query ID. */
             int64_t queryId;
@@ -684,28 +566,82 @@ namespace ignite
         };
 
         /**
-         * Query fetch response.
+         * Query execute batch start response.
          */
-        class QueryFetchResponse : public Response
+        class QueryExecuteBatchResponse : public Response
         {
         public:
             /**
              * Constructor.
-             * @param resultPage Result page.
              */
-            QueryFetchResponse(ResultPage& resultPage) : queryId(0), resultPage(resultPage)
+            QueryExecuteBatchResponse();
+
+            /**
+             * Destructor.
+             */
+            virtual ~QueryExecuteBatchResponse();
+
+            /**
+             * Affected rows.
+             * @return Affected rows.
+             */
+            int64_t GetAffectedRows() const
             {
-                // No-op.
+                return affectedRows;
             }
 
             /**
-             * Destructor.
+             * Get index of the set which caused an error.
+             * @return Index of the set which caused an error.
              */
-            ~QueryFetchResponse()
+            int64_t GetErrorSetIdx() const
             {
-                // No-op.
+                return affectedRows;
+            }
+
+            /**
+             * Get error message.
+             * @return Error message.
+             */
+            const std::string& GetErrorMessage() const
+            {
+                return errorMessage;
             }
 
+        private:
+            /**
+             * Read response using provided reader.
+             * @param reader Reader.
+             */
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
+
+            /** Affected rows. */
+            int64_t affectedRows;
+
+            /** Index of the set which caused an error. */
+            int64_t errorSetIdx;
+
+            /** Error message. */
+            std::string errorMessage;
+        };
+
+        /**
+         * Query fetch response.
+         */
+        class QueryFetchResponse : public Response
+        {
+        public:
+            /**
+             * Constructor.
+             * @param resultPage Result page.
+             */
+            QueryFetchResponse(ResultPage& resultPage);
+
+            /**
+             * Destructor.
+             */
+            virtual ~QueryFetchResponse();
+
             /**
              * Get query ID.
              * @return Query ID.
@@ -720,12 +656,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                queryId = reader.ReadInt64();
-
-                resultPage.Read(reader);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Query ID. */
             int64_t queryId;
@@ -743,18 +674,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryGetColumnsMetaResponse()
-            {
-                // No-op.
-            }
+            QueryGetColumnsMetaResponse();
 
             /**
              * Destructor.
              */
-            ~QueryGetColumnsMetaResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryGetColumnsMetaResponse();
 
             /**
              * Get column metadata.
@@ -770,10 +695,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                meta::ReadColumnMetaVector(reader, meta);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Columns metadata. */
             meta::ColumnMetaVector meta;
@@ -788,18 +710,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryGetTablesMetaResponse()
-            {
-                // No-op.
-            }
+            QueryGetTablesMetaResponse();
 
             /**
              * Destructor.
              */
-            ~QueryGetTablesMetaResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryGetTablesMetaResponse();
 
             /**
              * Get column metadata.
@@ -815,10 +731,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                meta::ReadTableMetaVector(reader, meta);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Columns metadata. */
             meta::TableMetaVector meta;
@@ -833,18 +746,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryGetParamsMetaResponse()
-            {
-                // No-op.
-            }
+            QueryGetParamsMetaResponse();
 
             /**
              * Destructor.
              */
-            ~QueryGetParamsMetaResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryGetParamsMetaResponse();
 
             /**
              * Get parameter type IDs.
@@ -860,10 +767,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                utility::ReadByteArray(reader, typeIds);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Columns metadata. */
             std::vector<int8_t> typeIds;
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
new file mode 100644
index 0000000000000..a691f7351d956
--- /dev/null
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _IGNITE_ODBC_QUERY_BATCH_QUERY
+#define _IGNITE_ODBC_QUERY_BATCH_QUERY
+
+#include "ignite/odbc/query/query.h"
+#include "ignite/odbc/app/parameter_set.h"
+#include "ignite/odbc/cursor.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        /** Connection forward-declaration. */
+        class Connection;
+
+        namespace query
+        {
+            /**
+             * Query.
+             */
+            class BatchQuery : public Query
+            {
+            public:
+                /**
+                 * Constructor.
+                 *
+                 * @param diag Diagnostics collector.
+                 * @param connection Associated connection.
+                 * @param sql SQL query string.
+                 * @param params SQL params.
+                 */
+                BatchQuery(diagnostic::Diagnosable& diag, Connection& connection,
+                    const std::string& sql, const app::ParameterSet& params);
+
+                /**
+                 * Destructor.
+                 */
+                virtual ~BatchQuery();
+
+                /**
+                 * Execute query.
+                 *
+                 * @return True on success.
+                 */
+                virtual SqlResult Execute();
+
+                /**
+                 * Get column metadata.
+                 *
+                 * @return Column metadata.
+                 */
+                virtual const meta::ColumnMetaVector& GetMeta() const;
+
+                /**
+                 * Fetch next result row to application buffers.
+                 *
+                 * @param columnBindings Application buffers to put data to.
+                 * @return Operation result.
+                 */
+                virtual SqlResult FetchNextRow(app::ColumnBindingMap& columnBindings);
+                
+                /**
+                 * Get data of the specified column in the result set.
+                 *
+                 * @param columnIdx Column index.
+                 * @param buffer Buffer to put column data to.
+                 * @return Operation result.
+                 */
+                virtual SqlResult GetColumn(uint16_t columnIdx, app::ApplicationDataBuffer& buffer);
+
+                /**
+                 * Close query.
+                 *
+                 * @return Result.
+                 */
+                virtual SqlResult Close();
+
+                /**
+                 * Check if data is available.
+                 *
+                 * @return True if data is available.
+                 */
+                virtual bool DataAvailable() const;
+
+                /**
+                 * Get number of rows affected by the statement.
+                 *
+                 * @return Number of rows affected by the statement.
+                 */
+                virtual int64_t AffectedRows() const;
+
+                /**
+                 * Get SQL query string.
+                 *
+                 * @return SQL query string.
+                 */
+                const std::string& GetSql() const
+                {
+                    return sql;
+                }
+
+            private:
+                IGNITE_NO_COPY_ASSIGNMENT(BatchQuery);
+
+                /**
+                 * Make query execute request and use response to set internal
+                 * state.
+                 *
+                 * @param begin Paramset interval beginning.
+                 * @param end Paramset interval end.
+                 * @param last Last page flag.
+                 * @return Result.
+                 */
+                SqlResult MakeRequestExecuteBatch(SqlUlen begin, SqlUlen end, bool last);
+
+                /** Connection associated with the statement. */
+                Connection& connection;
+
+                /** SQL Query. */
+                std::string sql;
+
+                /** Parameter bindings. */
+                const app::ParameterSet& params;
+
+                /** Columns metadata. */
+                meta::ColumnMetaVector resultMeta;
+
+                /** Number of rows affected. */
+                int64_t rowsAffected;
+
+                /** Number of parameter sets successfully processed. */
+                int64_t setsProcessed;
+
+                /** Query executed. */
+                bool executed;
+
+                /** Data retrieved. */
+                bool dataRetrieved;
+            };
+        }
+    }
+}
+
+#endif //_IGNITE_ODBC_QUERY_BATCH_QUERY
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
index 68bb8776417c8..0424bf8364a6d 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
@@ -19,7 +19,7 @@
 #define _IGNITE_ODBC_QUERY_DATA_QUERY
 
 #include "ignite/odbc/query/query.h"
-#include "ignite/odbc/app/parameter.h"
+#include "ignite/odbc/app/parameter_set.h"
 #include "ignite/odbc/cursor.h"
 
 namespace ignite
@@ -46,7 +46,7 @@ namespace ignite
                  * @param params SQL params.
                  */
                 DataQuery(diagnostic::Diagnosable& diag, Connection& connection,
-                    const std::string& sql, const app::ParameterBindingMap& params);
+                    const std::string& sql, const app::ParameterSet& params);
 
                 /**
                  * Destructor.
@@ -122,24 +122,32 @@ namespace ignite
                  * Make query execute request and use response to set internal
                  * state.
                  *
-                 * @return True on success.
+                 * @return Result.
                  */
                 SqlResult MakeRequestExecute();
 
                 /**
                  * Make query close request.
                  *
-                 * @return True on success.
+                 * @return Result.
                  */
                 SqlResult MakeRequestClose();
 
                 /**
                  * Make data fetch request and use response to set internal state.
                  *
-                 * @return True on success.
+                 * @return Result.
                  */
                 SqlResult MakeRequestFetch();
 
+                /**
+                 * Close query.
+                 * Non-virtual implementation.
+                 *
+                 * @return True on success.
+                 */
+                SqlResult InternalClose();
+
                 /** Connection associated with the statement. */
                 Connection& connection;
 
@@ -147,7 +155,7 @@ namespace ignite
                 std::string sql;
 
                 /** Parameter bindings. */
-                const app::ParameterBindingMap& params;
+                const app::ParameterSet& params;
 
                 /** Columns metadata. */
                 meta::ColumnMetaVector resultMeta;
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
index 40be1ed4c8bd6..d8103af182a30 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
@@ -47,6 +47,9 @@ namespace ignite
                     /** Data query type. */
                     DATA,
 
+                    /** Batch query type. */
+                    BATCH,
+
                     /** Foreign keys query type. */
                     FOREIGN_KEYS,
 
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
index d337d0311b972..ee37bb012bb33 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
@@ -95,7 +95,7 @@ namespace ignite
                  * @return Number of rows affected by the statement.
                  */
                 virtual int64_t AffectedRows() const;
-                
+
             private:
                 IGNITE_NO_COPY_ASSIGNMENT(TypeInfoQuery);
 
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
index 1ee56197ce81b..170f59998e461 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
@@ -23,19 +23,12 @@
 #include <map>
 #include <memory>
 
-#include <ignite/impl/interop/interop_output_stream.h>
-#include <ignite/impl/interop/interop_input_stream.h>
-#include <ignite/impl/binary/binary_writer_impl.h>
-
 #include "ignite/odbc/meta/column_meta.h"
-#include "ignite/odbc/meta/table_meta.h"
 #include "ignite/odbc/query/query.h"
 #include "ignite/odbc/app/application_data_buffer.h"
-#include "ignite/odbc/app/parameter.h"
+#include "ignite/odbc/app/parameter_set.h"
 #include "ignite/odbc/diagnostic/diagnosable_adapter.h"
 #include "ignite/odbc/common_types.h"
-#include "ignite/odbc/cursor.h"
-#include "ignite/odbc/utility.h"
 
 namespace ignite
 {
@@ -96,14 +89,23 @@ namespace ignite
              * @return Columns number.
              */
             int32_t GetColumnNumber();
-
+            
             /**
              * Bind parameter.
              *
              * @param paramIdx Parameter index.
-             * @param param Parameter.
+             * @param ioType Type of the parameter (input/output).
+             * @param bufferType The data type of the parameter.
+             * @param paramSqlType The SQL data type of the parameter.
+             * @param columnSize  The size of the column or expression of the corresponding parameter marker.
+             * @param decDigits  The decimal digits of the column or expression of the corresponding parameter marker.
+             * @param buffer A pointer to a buffer for the parameter's data.
+             * @param bufferLen Length of the ParameterValuePtr buffer in bytes.
+             * @param resLen A pointer to a buffer for the parameter's length.
+             * @return Operation result.
              */
-            void BindParameter(uint16_t paramIdx, const app::Parameter& param);
+            void BindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType, int16_t paramSqlType,
+                SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen);
 
             /**
              * Unbind specified parameter.
@@ -150,13 +152,6 @@ namespace ignite
              */
             void SetParamBindOffsetPtr(int* ptr);
 
-            /**
-             * Get parameter binding offset pointer.
-             *
-             * @return Parameter binding offset pointer.
-             */
-            int* GetParamBindOffsetPtr();
-
             /**
              * Get value of the column in the result set.
              *
@@ -171,7 +166,7 @@ namespace ignite
              * @param query SQL query.
              */
             void PrepareSqlQuery(const std::string& query);
-            
+
             /**
              * Execute SQL query.
              *
@@ -366,10 +361,18 @@ namespace ignite
              * Bind parameter.
              *
              * @param paramIdx Parameter index.
-             * @param param Parameter.
+             * @param ioType Type of the parameter (input/output).
+             * @param bufferType The data type of the parameter.
+             * @param paramSqlType The SQL data type of the parameter.
+             * @param columnSize  The size of the column or expression of the corresponding parameter marker.
+             * @param decDigits  The decimal digits of the column or expression of the corresponding parameter marker.
+             * @param buffer A pointer to a buffer for the parameter's data.
+             * @param bufferLen Length of the ParameterValuePtr buffer in bytes.
+             * @param resLen A pointer to a buffer for the parameter's length.
              * @return Operation result.
              */
-            SqlResult InternalBindParameter(uint16_t paramIdx, const app::Parameter& param);
+            SqlResult InternalBindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType, int16_t paramSqlType,
+                SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen);
 
             /**
              * Set statement attribute.
@@ -425,7 +428,7 @@ namespace ignite
              * @return Operation result.
              */
             SqlResult InternalPrepareSqlQuery(const std::string& query);
-            
+
             /**
              * Execute SQL query.
              *
@@ -611,12 +614,6 @@ namespace ignite
             /** Column bindings. */
             app::ColumnBindingMap columnBindings;
 
-            /** Parameter bindings. */
-            app::ParameterBindingMap paramBindings;
-
-            /** Parameter meta. */
-            std::vector<int8_t> paramTypes;
-
             /** Underlying query. */
             std::auto_ptr<query::Query> currentQuery;
 
@@ -626,14 +623,10 @@ namespace ignite
             /** Array to store statuses of rows fetched by the last fetch. */
             uint16_t* rowStatuses;
 
-            /** Offset added to pointers to change binding of parameters. */
-            int* paramBindOffset;
-            
             /** Offset added to pointers to change binding of column data. */
             int* columnBindOffset;
 
-            /** Index of the parameter, which is currently being set. */
-            uint16_t currentParamIdx;
+            app::ParameterSet parameters;
         };
     }
 }
diff --git a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
index 56358c5f8bf4f..280665984e50f 100644
--- a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
+++ b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
@@ -160,6 +160,7 @@
     <ClCompile Include="..\..\os\win\src\system\ui\window.cpp" />
     <ClCompile Include="..\..\src\app\application_data_buffer.cpp" />
     <ClCompile Include="..\..\src\app\parameter.cpp" />
+    <ClCompile Include="..\..\src\app\parameter_set.cpp" />
     <ClCompile Include="..\..\src\column.cpp" />
     <ClCompile Include="..\..\src\common_types.cpp" />
     <ClCompile Include="..\..\src\config\configuration.cpp" />
@@ -172,10 +173,12 @@
     <ClCompile Include="..\..\src\dsn_config.cpp" />
     <ClCompile Include="..\..\src\entry_points.cpp" />
     <ClCompile Include="..\..\src\environment.cpp" />
+    <ClCompile Include="..\..\src\message.cpp" />
     <ClCompile Include="..\..\src\meta\column_meta.cpp" />
     <ClCompile Include="..\..\src\meta\table_meta.cpp" />
     <ClCompile Include="..\..\src\odbc.cpp" />
     <ClCompile Include="..\..\src\protocol_version.cpp" />
+    <ClCompile Include="..\..\src\query\batch_query.cpp" />
     <ClCompile Include="..\..\src\query\data_query.cpp" />
     <ClCompile Include="..\..\src\query\column_metadata_query.cpp" />
     <ClCompile Include="..\..\src\query\foreign_keys_query.cpp" />
@@ -196,6 +199,7 @@
     <ClInclude Include="..\..\include\ignite\odbc.h" />
     <ClInclude Include="..\..\include\ignite\odbc\app\application_data_buffer.h" />
     <ClInclude Include="..\..\include\ignite\odbc\app\parameter.h" />
+    <ClInclude Include="..\..\include\ignite\odbc\app\parameter_set.h" />
     <ClInclude Include="..\..\include\ignite\odbc\column.h" />
     <ClInclude Include="..\..\include\ignite\odbc\common_types.h" />
     <ClInclude Include="..\..\include\ignite\odbc\config\configuration.h" />
@@ -214,6 +218,7 @@
     <ClInclude Include="..\..\include\ignite\odbc\meta\table_meta.h" />
     <ClInclude Include="..\..\include\ignite\odbc\parser.h" />
     <ClInclude Include="..\..\include\ignite\odbc\protocol_version.h" />
+    <ClInclude Include="..\..\include\ignite\odbc\query\batch_query.h" />
     <ClInclude Include="..\..\include\ignite\odbc\query\data_query.h" />
     <ClInclude Include="..\..\include\ignite\odbc\query\column_metadata_query.h" />
     <ClInclude Include="..\..\include\ignite\odbc\query\foreign_keys_query.h" />
diff --git a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
index 58764e4043871..9aefc0c616ea1 100644
--- a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
+++ b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
@@ -136,6 +136,15 @@
     <ClCompile Include="..\..\src\protocol_version.cpp">
       <Filter>Code</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\app\parameter_set.cpp">
+      <Filter>Code\app</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\query\batch_query.cpp">
+      <Filter>Code\query</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\message.cpp">
+      <Filter>Code</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="module.def">
@@ -260,5 +269,11 @@
     <ClInclude Include="..\..\include\ignite\odbc\protocol_version.h">
       <Filter>Code</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\ignite\odbc\app\parameter_set.h">
+      <Filter>Code\app</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\include\ignite\odbc\query\batch_query.h">
+      <Filter>Code\query</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp b/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp
index 71c5f39025796..c13857e815c51 100644
--- a/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp
+++ b/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp
@@ -33,35 +33,38 @@ namespace ignite
     {
         namespace app
         {
-            using ignite::impl::binary::BinaryUtils;
+            using impl::binary::BinaryUtils;
 
             ApplicationDataBuffer::ApplicationDataBuffer() :
                 type(type_traits::IGNITE_ODBC_C_TYPE_UNSUPPORTED),
                 buffer(0),
                 buflen(0),
                 reslen(0),
-                offset(0)
+                byteOffset(0),
+                elementOffset(0)
             {
                 // No-op.
             }
 
-            ApplicationDataBuffer::ApplicationDataBuffer(type_traits::IgniteSqlType type, 
-                void* buffer, SqlLen buflen, SqlLen* reslen, int** offset) :
+            ApplicationDataBuffer::ApplicationDataBuffer(type_traits::IgniteSqlType type,
+                void* buffer, SqlLen buflen, SqlLen* reslen) :
                 type(type),
                 buffer(buffer),
                 buflen(buflen),
                 reslen(reslen),
-                offset(offset)
+                byteOffset(0),
+                elementOffset(0)
             {
                 // No-op.
             }
 
-            ApplicationDataBuffer::ApplicationDataBuffer(const ApplicationDataBuffer & other) :
+            ApplicationDataBuffer::ApplicationDataBuffer(const ApplicationDataBuffer& other) :
                 type(other.type),
                 buffer(other.buffer),
                 buflen(other.buflen),
                 reslen(other.reslen),
-                offset(other.offset)
+                byteOffset(other.byteOffset),
+                elementOffset(other.elementOffset)
             {
                 // No-op.
             }
@@ -77,7 +80,8 @@ namespace ignite
                 buffer = other.buffer;
                 buflen = other.buflen;
                 reslen = other.reslen;
-                offset = other.offset;
+                byteOffset = other.byteOffset;
+                elementOffset = other.elementOffset;
 
                 return *this;
             }
@@ -1072,22 +1076,22 @@ namespace ignite
 
             const void* ApplicationDataBuffer::GetData() const
             {
-                return ApplyOffset(buffer);
+                return ApplyOffset(buffer, GetElementSize());
             }
 
             const SqlLen* ApplicationDataBuffer::GetResLen() const
             {
-                return ApplyOffset(reslen);
+                return ApplyOffset(reslen, sizeof(*reslen));
             }
 
-            void* ApplicationDataBuffer::GetData() 
+            void* ApplicationDataBuffer::GetData()
             {
-                return ApplyOffset(buffer);
+                return ApplyOffset(buffer, GetElementSize());
             }
 
             SqlLen* ApplicationDataBuffer::GetResLen()
             {
-                return ApplyOffset(reslen);
+                return ApplyOffset(reslen, sizeof(*reslen));
             }
 
             template<typename T>
@@ -1407,12 +1411,12 @@ namespace ignite
             }
 
             template<typename T>
-            T* ApplicationDataBuffer::ApplyOffset(T* ptr) const
+            T* ApplicationDataBuffer::ApplyOffset(T* ptr, size_t elemSize) const
             {
-                if (!ptr || !offset || !*offset)
+                if (!ptr)
                     return ptr;
 
-                return utility::GetPointerWithOffset(ptr, **offset);
+                return utility::GetPointerWithOffset(ptr, byteOffset + elemSize * elementOffset);
             }
 
             bool ApplicationDataBuffer::IsDataAtExec() const
@@ -1502,6 +1506,64 @@ namespace ignite
                 return 0;
             }
 
+            SqlLen ApplicationDataBuffer::GetElementSize() const
+            {
+                using namespace type_traits;
+
+                switch (type)
+                {
+                    case IGNITE_ODBC_C_TYPE_WCHAR:
+                    case IGNITE_ODBC_C_TYPE_CHAR:
+                    case IGNITE_ODBC_C_TYPE_BINARY:
+                        return buflen;
+
+                    case IGNITE_ODBC_C_TYPE_SIGNED_SHORT:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT:
+                        return static_cast<SqlLen>(sizeof(short));
+
+                    case IGNITE_ODBC_C_TYPE_SIGNED_LONG:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_LONG:
+                        return static_cast<SqlLen>(sizeof(long));
+
+                    case IGNITE_ODBC_C_TYPE_FLOAT:
+                        return static_cast<SqlLen>(sizeof(float));
+
+                    case IGNITE_ODBC_C_TYPE_DOUBLE:
+                        return static_cast<SqlLen>(sizeof(double));
+
+                    case IGNITE_ODBC_C_TYPE_BIT:
+                    case IGNITE_ODBC_C_TYPE_SIGNED_TINYINT:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_TINYINT:
+                        return static_cast<SqlLen>(sizeof(char));
+
+                    case IGNITE_ODBC_C_TYPE_SIGNED_BIGINT:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT:
+                        return static_cast<SqlLen>(sizeof(SQLBIGINT));
+
+                    case IGNITE_ODBC_C_TYPE_TDATE:
+                        return static_cast<SqlLen>(sizeof(SQL_DATE_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_TTIME:
+                        return static_cast<SqlLen>(sizeof(SQL_TIME_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_TTIMESTAMP:
+                        return static_cast<SqlLen>(sizeof(SQL_TIMESTAMP_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_NUMERIC:
+                        return static_cast<SqlLen>(sizeof(SQL_NUMERIC_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_GUID:
+                        return static_cast<SqlLen>(sizeof(SQLGUID));
+
+                    case IGNITE_ODBC_C_TYPE_DEFAULT:
+                    case IGNITE_ODBC_C_TYPE_UNSUPPORTED:
+                    default:
+                        break;
+                }
+
+                return 0;
+            }
+
             SqlLen ApplicationDataBuffer::GetInputSize() const
             {
                 if (!IsDataAtExec())
diff --git a/modules/platforms/cpp/odbc/src/app/parameter.cpp b/modules/platforms/cpp/odbc/src/app/parameter.cpp
index 937ef58e7c6c3..5ee132c1b7107 100644
--- a/modules/platforms/cpp/odbc/src/app/parameter.cpp
+++ b/modules/platforms/cpp/odbc/src/app/parameter.cpp
@@ -16,8 +16,6 @@
  */
 
 #include <algorithm>
-#include <string>
-#include <sstream>
 
 #include "ignite/odbc/system/odbc_constants.h"
 #include "ignite/odbc/app/parameter.h"
@@ -78,7 +76,7 @@ namespace ignite
                 return *this;
             }
 
-            void Parameter::Write(ignite::impl::binary::BinaryWriterImpl& writer) const
+            void Parameter::Write(impl::binary::BinaryWriterImpl& writer, int offset, SqlUlen idx) const
             {
                 if (buffer.GetInputSize() == SQL_NULL_DATA)
                 {
@@ -89,6 +87,8 @@ namespace ignite
 
                 // Buffer to use to get data.
                 ApplicationDataBuffer buf(buffer);
+                buf.SetByteOffset(offset);
+                buf.SetElementOffset(idx);
 
                 SqlLen storedDataLen = static_cast<SqlLen>(storedData.size());
 
@@ -150,12 +150,14 @@ namespace ignite
                         break;
                     }
 
+                    case SQL_TYPE_DATE:
                     case SQL_DATE:
                     {
                         writer.WriteDate(buf.GetDate());
                         break;
                     }
 
+                    case SQL_TYPE_TIMESTAMP:
                     case SQL_TIMESTAMP:
                     {
                         writer.WriteTimestamp(buf.GetTimestamp());
@@ -207,6 +209,11 @@ namespace ignite
                 return buffer;
             }
 
+            const ApplicationDataBuffer& Parameter::GetBuffer() const
+            {
+                return buffer;
+            }
+
             void Parameter::ResetStoredData()
             {
                 storedData.clear();
diff --git a/modules/platforms/cpp/odbc/src/app/parameter_set.cpp b/modules/platforms/cpp/odbc/src/app/parameter_set.cpp
new file mode 100644
index 0000000000000..c110d05dc5b38
--- /dev/null
+++ b/modules/platforms/cpp/odbc/src/app/parameter_set.cpp
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ignite/odbc/app/parameter_set.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        namespace app
+        {
+            ParameterSet::ParameterSet():
+                parameters(),
+                paramTypes(),
+                paramBindOffset(0),
+                processedParamRows(0),
+                paramSetSize(1),
+                paramSetPos(0),
+                currentParamIdx(0),
+                typesSet(false)
+            {
+                // No-op.
+            }
+
+            void ParameterSet::SetParamSetSize(SqlUlen size)
+            {
+                paramSetSize = size;
+            }
+
+            void ParameterSet::BindParameter(uint16_t paramIdx, const Parameter& param)
+            {
+                parameters[paramIdx] = param;
+            }
+
+            void ParameterSet::UnbindParameter(uint16_t paramIdx)
+            {
+                parameters.erase(paramIdx);
+            }
+
+            void ParameterSet::UnbindAll()
+            {
+                parameters.clear();
+            }
+
+            uint16_t ParameterSet::GetParametersNumber() const
+            {
+                return static_cast<uint16_t>(parameters.size());
+            }
+
+            void ParameterSet::SetParamBindOffsetPtr(int* ptr)
+            {
+                paramBindOffset = ptr;
+            }
+
+            int* ParameterSet::GetParamBindOffsetPtr()
+            {
+                return paramBindOffset;
+            }
+
+            void ParameterSet::Prepare()
+            {
+                paramTypes.clear();
+
+                typesSet = false;
+
+                paramSetPos = 0;
+
+                for (ParameterBindingMap::iterator it = parameters.begin(); it != parameters.end(); ++it)
+                    it->second.ResetStoredData();
+            }
+
+            bool ParameterSet::IsDataAtExecNeeded() const
+            {
+                for (ParameterBindingMap::const_iterator it = parameters.begin(); it != parameters.end(); ++it)
+                {
+                    if (!it->second.IsDataReady())
+                        return true;
+                }
+
+                return false;
+            }
+
+            void ParameterSet::SetParamsProcessedPtr(SqlUlen* ptr)
+            {
+                processedParamRows = ptr;
+            }
+
+            SqlUlen* ParameterSet::GetParamsProcessedPtr()
+            {
+                return processedParamRows;
+            }
+
+            void ParameterSet::SetParamsProcessed(SqlUlen processed) const
+            {
+                if (processedParamRows)
+                    *processedParamRows = processed;
+            }
+
+            void ParameterSet::UpdateParamsTypes(const ParameterTypeVector& meta)
+            {
+                paramTypes = meta;
+
+                typesSet = true;
+            }
+
+            int8_t ParameterSet::GetParamType(int16_t idx, int8_t dflt)
+            {
+                if (idx > 0 && static_cast<size_t>(idx) <= paramTypes.size())
+                    return paramTypes[idx - 1];
+
+                return dflt;
+            }
+
+            uint16_t ParameterSet::GetExpectedParamNum()
+            {
+                return static_cast<uint16_t>(paramTypes.size());
+            }
+
+            bool ParameterSet::IsMetadataSet() const
+            {
+                return typesSet;
+            }
+
+            bool ParameterSet::IsParameterSelected() const
+            {
+                return currentParamIdx != 0;
+            }
+
+            Parameter* ParameterSet::GetParameter(uint16_t idx)
+            {
+                ParameterBindingMap::iterator it = parameters.find(currentParamIdx);
+
+                if (it != parameters.end())
+                    return &it->second;
+
+                return 0;
+            }
+
+            Parameter* ParameterSet::GetSelectedParameter()
+            {
+                return GetParameter(currentParamIdx);
+            }
+
+            Parameter* ParameterSet::SelectNextParameter()
+            {
+                for (ParameterBindingMap::iterator it = parameters.begin(); it != parameters.end(); ++it)
+                {
+                    uint16_t paramIdx = it->first;
+                    Parameter& param = it->second;
+
+                    if (!param.IsDataReady())
+                    {
+                        currentParamIdx = paramIdx;
+
+                        return &param;
+                    }
+                }
+
+                return 0;
+            }
+
+            void ParameterSet::Write(impl::binary::BinaryWriterImpl& writer) const
+            {
+                writer.WriteInt32(CalculateRowLen());
+
+                WriteRow(writer, 0);
+            }
+
+            void ParameterSet::Write(impl::binary::BinaryWriterImpl& writer, SqlUlen begin, SqlUlen end, bool last) const
+            {
+                int32_t rowLen = CalculateRowLen();
+
+                writer.WriteInt32(rowLen);
+
+                SqlUlen intervalEnd = std::min(paramSetSize, end);
+
+                assert(begin < intervalEnd);
+
+                int32_t intervalLen = static_cast<int32_t>(intervalEnd - begin);
+
+                writer.WriteInt32(intervalLen);
+                writer.WriteBool(last);
+
+                if (rowLen)
+                {
+                    for (SqlUlen i = begin; i < intervalEnd; ++i)
+                        WriteRow(writer, i);
+                }
+            }
+
+            void ParameterSet::WriteRow(impl::binary::BinaryWriterImpl& writer, SqlUlen idx) const
+            {
+                uint16_t prev = 0;
+
+                int appOffset = paramBindOffset ? *paramBindOffset : 0;
+
+                for (ParameterBindingMap::const_iterator it = parameters.begin(); it != parameters.end(); ++it)
+                {
+                    uint16_t paramIdx = it->first;
+                    const Parameter& param = it->second;
+
+                    while ((paramIdx - prev) > 1)
+                    {
+                        writer.WriteNull();
+                        ++prev;
+                    }
+
+                    param.Write(writer, appOffset, idx);
+
+                    prev = paramIdx;
+                }
+            }
+
+            int32_t ParameterSet::CalculateRowLen() const
+            {
+                if (!parameters.empty())
+                    return static_cast<int32_t>(parameters.rbegin()->first);
+
+                return  0;
+            }
+
+            int32_t ParameterSet::GetParamSetSize() const
+            {
+                return static_cast<int32_t>(paramSetSize);
+            }
+        }
+    }
+}
diff --git a/modules/platforms/cpp/odbc/src/config/connection_info.cpp b/modules/platforms/cpp/odbc/src/config/connection_info.cpp
index 341ab7fea2863..6fe03f24dcf40 100644
--- a/modules/platforms/cpp/odbc/src/config/connection_info.cpp
+++ b/modules/platforms/cpp/odbc/src/config/connection_info.cpp
@@ -29,11 +29,11 @@
 
 #ifndef SQL_ASYNC_NOTIFICATION_NOT_CAPABLE
 #define SQL_ASYNC_NOTIFICATION_NOT_CAPABLE      0x00000000L
-#endif 
+#endif
 
 #ifndef SQL_ASYNC_NOTIFICATION_CAPABLE
 #define SQL_ASYNC_NOTIFICATION_CAPABLE          0x00000001L
-#endif 
+#endif
 
 namespace ignite
 {
@@ -95,7 +95,9 @@ namespace ignite
                     DBG_STR_CASE(SQL_SQL92_VALUE_EXPRESSIONS);
                     DBG_STR_CASE(SQL_STATIC_CURSOR_ATTRIBUTES1);
                     DBG_STR_CASE(SQL_STATIC_CURSOR_ATTRIBUTES2);
-                default: 
+                    DBG_STR_CASE(SQL_PARAM_ARRAY_ROW_COUNTS);
+                    DBG_STR_CASE(SQL_PARAM_ARRAY_SELECTS);
+                default:
                     break;
                 }
                 return "<< UNKNOWN TYPE >>";
@@ -117,7 +119,7 @@ namespace ignite
                 strParams[SQL_DBMS_VER]        = "03.00";
 
 #ifdef SQL_DRIVER_VER
-                // Driver version. At a minimum, the version is of the form 
+                // Driver version. At a minimum, the version is of the form
                 // ##.##.####, where the first two digits are the major version,
                 // the next two digits are the minor version, and the last four
                 // digits are the release version.
@@ -125,7 +127,7 @@ namespace ignite
 #endif // SQL_DRIVER_VER
 
 #ifdef SQL_COLUMN_ALIAS
-                // A character string: "Y" if the data source supports column 
+                // A character string: "Y" if the data source supports column
                 // aliases; otherwise, "N".
                 strParams[SQL_COLUMN_ALIAS] = "Y";
 #endif // SQL_COLUMN_ALIAS
@@ -168,7 +170,7 @@ namespace ignite
 #endif // SQL_TABLE_TERM
 
 #ifdef SQL_SCHEMA_TERM
-                // A character string with the data source vendor's name for 
+                // A character string with the data source vendor's name for
                 // a schema; for example, "owner", "Authorization ID", or "Schema".
                 strParams[SQL_SCHEMA_TERM] = "schema";
 #endif // SQL_SCHEMA_TERM
@@ -194,9 +196,9 @@ namespace ignite
 
 #ifdef SQL_ASYNC_NOTIFICATION
                 // Indicates if the driver supports asynchronous notification.
-                // SQL_ASYNC_NOTIFICATION_CAPABLE  = Asynchronous execution 
+                // SQL_ASYNC_NOTIFICATION_CAPABLE  = Asynchronous execution
                 // notification is supported by the driver.
-                // SQL_ASYNC_NOTIFICATION_NOT_CAPABLE Asynchronous execution 
+                // SQL_ASYNC_NOTIFICATION_NOT_CAPABLE Asynchronous execution
                 // notification is not supported by the driver.
                 intParams[SQL_ASYNC_NOTIFICATION] = SQL_ASYNC_NOTIFICATION_NOT_CAPABLE;
 #endif // SQL_ASYNC_NOTIFICATION
@@ -207,7 +209,7 @@ namespace ignite
 #endif // SQL_GETDATA_EXTENSIONS
 
 #ifdef SQL_ODBC_INTERFACE_CONFORMANCE
-                // Indicates the level of the ODBC 3.x interface that the driver 
+                // Indicates the level of the ODBC 3.x interface that the driver
                 // complies with.
                 intParams[SQL_ODBC_INTERFACE_CONFORMANCE] = SQL_OIC_CORE;
 #endif // SQL_ODBC_INTERFACE_CONFORMANCE
@@ -229,7 +231,7 @@ namespace ignite
 #endif // SQL_SCHEMA_USAGE
 
 #ifdef SQL_MAX_IDENTIFIER_LEN
-                // Indicates the maximum size in characters that the data source 
+                // Indicates the maximum size in characters that the data source
                 // supports for user-defined names.
                 intParams[SQL_MAX_IDENTIFIER_LEN] = 128;
 #endif // SQL_MAX_IDENTIFIER_LEN
@@ -243,7 +245,7 @@ namespace ignite
 #ifdef SQL_NUMERIC_FUNCTIONS
                 // Bitmask enumerating the scalar numeric functions supported by
                 // the driver and associated data source.
-                intParams[SQL_NUMERIC_FUNCTIONS] = SQL_FN_NUM_ABS | SQL_FN_NUM_ACOS | SQL_FN_NUM_ASIN | 
+                intParams[SQL_NUMERIC_FUNCTIONS] = SQL_FN_NUM_ABS | SQL_FN_NUM_ACOS | SQL_FN_NUM_ASIN |
                     SQL_FN_NUM_ATAN | SQL_FN_NUM_ATAN2 | SQL_FN_NUM_CEILING | SQL_FN_NUM_COS | SQL_FN_NUM_COT |
                     SQL_FN_NUM_EXP | SQL_FN_NUM_FLOOR | SQL_FN_NUM_LOG | SQL_FN_NUM_MOD | SQL_FN_NUM_SIGN |
                     SQL_FN_NUM_SIN | SQL_FN_NUM_SQRT | SQL_FN_NUM_TAN | SQL_FN_NUM_PI | SQL_FN_NUM_RAND |
@@ -273,7 +275,7 @@ namespace ignite
 #endif // SQL_TIMEDATE_FUNCTIONS
 
 #ifdef SQL_TIMEDATE_ADD_INTERVALS
-                // Bitmask enumerating timestamp intervals supported by the driver 
+                // Bitmask enumerating timestamp intervals supported by the driver
                 // and associated data source for the TIMESTAMPADD scalar function.
                 intParams[SQL_TIMEDATE_ADD_INTERVALS] = 0;
 #endif // SQL_TIMEDATE_ADD_INTERVALS
@@ -303,7 +305,7 @@ namespace ignite
 #endif // SQL_CONVERT_FUNCTIONS
 
 #ifdef SQL_OJ_CAPABILITIES
-                // Bitmask enumerating the types of outer joins supported by the 
+                // Bitmask enumerating the types of outer joins supported by the
                 // driver and data source.
                 intParams[SQL_OJ_CAPABILITIES] = SQL_OJ_LEFT | SQL_OJ_NOT_ORDERED | SQL_OJ_ALL_COMPARISON_OPS;
 #endif // SQL_OJ_CAPABILITIES
@@ -333,7 +335,7 @@ namespace ignite
 #ifdef SQL_SQL92_VALUE_EXPRESSIONS
                 // Bitmask enumerating the value expressions supported,
                 // as defined in SQL-92.
-                intParams[SQL_SQL92_VALUE_EXPRESSIONS] = SQL_SVE_CASE | 
+                intParams[SQL_SQL92_VALUE_EXPRESSIONS] = SQL_SVE_CASE |
                     SQL_SVE_CAST | SQL_SVE_COALESCE | SQL_SVE_NULLIF;
 #endif // SQL_SQL92_VALUE_EXPRESSIONS
 
@@ -369,6 +371,40 @@ namespace ignite
                 intParams[SQL_STATIC_CURSOR_ATTRIBUTES2] = 0;
 #endif //SQL_STATIC_CURSOR_ATTRIBUTES2
 
+#ifdef SQL_PARAM_ARRAY_ROW_COUNTS
+                // Enumerating the driver's properties regarding the availability of row counts in a parameterized
+                // execution. Has the following values:
+                //
+                // SQL_PARC_BATCH = Individual row counts are available for each set of parameters. This is conceptually
+                //     equivalent to the driver generating a batch of SQL statements, one for each parameter set in the
+                //     array. Extended error information can be retrieved by using the SQL_PARAM_STATUS_PTR descriptor
+                //     field.
+                //
+                // SQL_PARC_NO_BATCH = There is only one row count available, which is the cumulative row count
+                //     resulting from the execution of the statement for the entire array of parameters. This is
+                //     conceptually equivalent to treating the statement together with the complete parameter array as
+                //     one atomic unit. Errors are handled the same as if one statement were executed.
+                intParams[SQL_PARAM_ARRAY_ROW_COUNTS] = SQL_PARC_NO_BATCH;
+#endif //SQL_PARAM_ARRAY_ROW_COUNTS
+
+#ifdef SQL_PARAM_ARRAY_SELECTS
+                // Enumerating the driver's properties regarding the availability of result sets in a parameterized
+                // execution. Has the following values:
+                //
+                // SQL_PAS_BATCH = There is one result set available per set of parameters. This is conceptually
+                //     equivalent to the driver generating a batch of SQL statements, one for each parameter set in
+                //     the array.
+                //
+                // SQL_PAS_NO_BATCH = There is only one result set available, which represents the cumulative result set
+                //     resulting from the execution of the statement for the complete array of parameters. This is
+                //     conceptually equivalent to treating the statement together with the complete parameter array as
+                //     one atomic unit.
+                //
+                // SQL_PAS_NO_SELECT = A driver does not allow a result - set generating statement to be executed with
+                //     an array of parameters.
+                intParams[SQL_PARAM_ARRAY_SELECTS] = SQL_PAS_NO_SELECT;
+#endif //SQL_PARAM_ARRAY_SELECTS
+
                 //======================= Short Params ========================
 #ifdef SQL_MAX_CONCURRENT_ACTIVITIES
                 // The maximum number of active statements that the driver can
@@ -412,10 +448,10 @@ namespace ignite
 
                 StringInfoMap::const_iterator itStr = strParams.find(type);
 
-                if (itStr != strParams.cend()) 
+                if (itStr != strParams.cend())
                 {
                     unsigned short strlen = static_cast<short>(
-                        utility::CopyStringToBuffer(itStr->second, 
+                        utility::CopyStringToBuffer(itStr->second,
                             reinterpret_cast<char*>(buf), buflen));
 
                     if (reslen)
diff --git a/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp b/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp
index 215d77f5c249f..6e873c296f1b8 100644
--- a/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp
+++ b/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp
@@ -75,6 +75,12 @@ namespace
 
     /** SQL state HYT01 constant. */
     const std::string STATE_HYT01 = "HYT01";
+
+    /** SQL state HY003 constant. */
+    const std::string STATE_HY003 = "HY003";
+
+    /** SQL state HY105 constant. */
+    const std::string STATE_HY105 = "HY105";
 }
 
 namespace ignite
@@ -246,6 +252,12 @@ namespace ignite
                     case SQL_STATE_HYT01_CONNECTIOIN_TIMEOUT:
                         return STATE_HYT01;
 
+                    case SQL_STATE_HY003_INVALID_APPLICATION_BUFFER_TYPE:
+                        return STATE_HY003;
+
+                    case SQL_STATE_HY105_INVALID_PARAMETER_TYPE:
+                        return STATE_HY105;
+
                     default:
                         break;
                 }
diff --git a/modules/platforms/cpp/odbc/src/message.cpp b/modules/platforms/cpp/odbc/src/message.cpp
new file mode 100644
index 0000000000000..741540526fb19
--- /dev/null
+++ b/modules/platforms/cpp/odbc/src/message.cpp
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ignite/odbc/message.h"
+#include "ignite/odbc/utility.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        HandshakeRequest::HandshakeRequest(int64_t version, bool distributedJoins, bool enforceJoinOrder):
+            version(version),
+            distributedJoins(distributedJoins),
+            enforceJoinOrder(enforceJoinOrder)
+        {
+            // No-op.
+        }
+
+        HandshakeRequest::~HandshakeRequest()
+        {
+            // No-op.
+        }
+
+        void HandshakeRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_HANDSHAKE);
+
+            writer.WriteInt64(version);
+
+            writer.WriteBool(distributedJoins);
+            writer.WriteBool(enforceJoinOrder);
+        }
+
+        QueryExecuteRequest::QueryExecuteRequest(const std::string& schema, const std::string& sql, const app::ParameterSet& params):
+            cache(schema),
+            sql(sql),
+            params(params)
+        {
+            // No-op.
+        }
+
+        QueryExecuteRequest::~QueryExecuteRequest()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_EXECUTE_SQL_QUERY);
+
+            if (cache.empty())
+                writer.WriteNull();
+            else
+                writer.WriteObject<std::string>(cache);
+
+            writer.WriteObject<std::string>(sql);
+
+            params.Write(writer);
+        }
+
+        QueryExecuteBatchtRequest::QueryExecuteBatchtRequest(const std::string& schema, const std::string& sql,
+            const app::ParameterSet& params, SqlUlen begin, SqlUlen end, bool last):
+            schema(schema),
+            sql(sql),
+            params(params),
+            begin(begin),
+            end(end),
+            last(last)
+        {
+            // No-op.
+        }
+
+        QueryExecuteBatchtRequest::~QueryExecuteBatchtRequest()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteBatchtRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_EXECUTE_SQL_QUERY_BATCH);
+
+            if (schema.empty())
+                writer.WriteNull();
+            else
+                writer.WriteObject<std::string>(schema);
+
+            writer.WriteObject<std::string>(sql);
+
+            params.Write(writer, begin, end, last);
+        }
+
+        QueryCloseRequest::QueryCloseRequest(int64_t queryId): queryId(queryId)
+        {
+            // No-op.
+        }
+
+        QueryCloseRequest::~QueryCloseRequest()
+        {
+            // No-op.
+        }
+
+        void QueryCloseRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_CLOSE_SQL_QUERY);
+            writer.WriteInt64(queryId);
+        }
+
+        QueryFetchRequest::QueryFetchRequest(int64_t queryId, int32_t pageSize):
+            queryId(queryId),
+            pageSize(pageSize)
+        {
+            // No-op.
+        }
+
+        QueryFetchRequest::~QueryFetchRequest()
+        {
+            // No-op.
+        }
+
+        void QueryFetchRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_FETCH_SQL_QUERY);
+            writer.WriteInt64(queryId);
+            writer.WriteInt32(pageSize);
+        }
+
+        QueryGetColumnsMetaRequest::QueryGetColumnsMetaRequest(const std::string& schema, const std::string& table, const std::string& column):
+            schema(schema),
+            table(table),
+            column(column)
+        {
+            // No-op.
+        }
+
+        QueryGetColumnsMetaRequest::~QueryGetColumnsMetaRequest()
+        {
+            // No-op.
+        }
+
+        void QueryGetColumnsMetaRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_GET_COLUMNS_METADATA);
+
+            utility::WriteString(writer, schema);
+            utility::WriteString(writer, table);
+            utility::WriteString(writer, column);
+        }
+
+        QueryGetTablesMetaRequest::QueryGetTablesMetaRequest(const std::string& catalog, const std::string& schema, const std::string& table, const std::string& tableTypes):
+            catalog(catalog),
+            schema(schema),
+            table(table),
+            tableTypes(tableTypes)
+        {
+            // No-op.
+        }
+
+        QueryGetTablesMetaRequest::~QueryGetTablesMetaRequest()
+        {
+            // No-op.
+        }
+
+        void QueryGetTablesMetaRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_GET_TABLES_METADATA);
+
+            utility::WriteString(writer, catalog);
+            utility::WriteString(writer, schema);
+            utility::WriteString(writer, table);
+            utility::WriteString(writer, tableTypes);
+        }
+
+        void QueryGetParamsMetaRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_GET_PARAMS_METADATA);
+
+            utility::WriteString(writer, cacheName);
+            utility::WriteString(writer, sqlQuery);
+        }
+
+        Response::Response():
+            status(RESPONSE_STATUS_FAILED),
+            error()
+        {
+            // No-op.
+        }
+
+        Response::~Response()
+        {
+            // No-op.
+        }
+
+        void Response::Read(impl::binary::BinaryReaderImpl& reader)
+        {
+            status = reader.ReadInt8();
+
+            if (status == RESPONSE_STATUS_SUCCESS)
+                ReadOnSuccess(reader);
+            else
+                utility::ReadString(reader, error);
+        }
+
+        void Response::ReadOnSuccess(impl::binary::BinaryReaderImpl&)
+        {
+            // No-op.
+        }
+
+        HandshakeResponse::HandshakeResponse():
+            accepted(false),
+            protoVerSince(),
+            currentVer()
+        {
+            // No-op.
+        }
+
+        HandshakeResponse::~HandshakeResponse()
+        {
+            // No-op.
+        }
+
+        void HandshakeResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            accepted = reader.ReadBool();
+
+            if (!accepted)
+            {
+                utility::ReadString(reader, protoVerSince);
+                utility::ReadString(reader, currentVer);
+            }
+        }
+
+        QueryCloseResponse::QueryCloseResponse(): queryId(0)
+        {
+            // No-op.
+        }
+
+        QueryCloseResponse::~QueryCloseResponse()
+        {
+            // No-op.
+        }
+
+        void QueryCloseResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            queryId = reader.ReadInt64();
+        }
+
+        QueryExecuteResponse::QueryExecuteResponse(): queryId(0), meta()
+        {
+            // No-op.
+        }
+
+        QueryExecuteResponse::~QueryExecuteResponse()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            queryId = reader.ReadInt64();
+
+            meta::ReadColumnMetaVector(reader, meta);
+        }
+
+        QueryExecuteBatchResponse::QueryExecuteBatchResponse():
+            affectedRows(0),
+            errorSetIdx(-1),
+            errorMessage()
+        {
+            // No-op.
+        }
+
+        QueryExecuteBatchResponse::~QueryExecuteBatchResponse()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteBatchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            bool success = reader.ReadBool();
+            affectedRows = reader.ReadInt64();
+
+            if (!success)
+            {
+                errorSetIdx = reader.ReadInt64();
+                errorMessage = reader.ReadObject<std::string>();
+            }
+        }
+
+        QueryFetchResponse::QueryFetchResponse(ResultPage& resultPage): queryId(0), resultPage(resultPage)
+        {
+            // No-op.
+        }
+
+        QueryFetchResponse::~QueryFetchResponse()
+        {
+            // No-op.
+        }
+
+        void QueryFetchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            queryId = reader.ReadInt64();
+
+            resultPage.Read(reader);
+        }
+
+        QueryGetColumnsMetaResponse::QueryGetColumnsMetaResponse()
+        {
+            // No-op.
+        }
+
+        QueryGetColumnsMetaResponse::~QueryGetColumnsMetaResponse()
+        {
+            // No-op.
+        }
+
+        void QueryGetColumnsMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            meta::ReadColumnMetaVector(reader, meta);
+        }
+
+        QueryGetTablesMetaResponse::QueryGetTablesMetaResponse()
+        {
+            // No-op.
+        }
+
+        QueryGetTablesMetaResponse::~QueryGetTablesMetaResponse()
+        {
+            // No-op.
+        }
+
+        void QueryGetTablesMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            meta::ReadTableMetaVector(reader, meta);
+        }
+
+        QueryGetParamsMetaResponse::QueryGetParamsMetaResponse()
+        {
+            // No-op.
+        }
+
+        QueryGetParamsMetaResponse::~QueryGetParamsMetaResponse()
+        {
+            // No-op.
+        }
+
+        void QueryGetParamsMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            utility::ReadByteArray(reader, typeIds);
+        }
+    }
+}
+
diff --git a/modules/platforms/cpp/odbc/src/odbc.cpp b/modules/platforms/cpp/odbc/src/odbc.cpp
index caf31d91860b1..5f4db6597bfea 100644
--- a/modules/platforms/cpp/odbc/src/odbc.cpp
+++ b/modules/platforms/cpp/odbc/src/odbc.cpp
@@ -630,27 +630,8 @@ namespace ignite
         if (!statement)
             return SQL_INVALID_HANDLE;
 
-        if (ioType != SQL_PARAM_INPUT)
-            return SQL_ERROR;
-
-        if (!IsSqlTypeSupported(paramSqlType))
-            return SQL_ERROR;
-
-        IgniteSqlType driverType = ToDriverType(bufferType);
-
-        if (driverType == IGNITE_ODBC_C_TYPE_UNSUPPORTED)
-            return SQL_ERROR;
-
-        if (buffer)
-        {
-            ApplicationDataBuffer dataBuffer(driverType, buffer, bufferLen, resLen);
-
-            Parameter param(dataBuffer, paramSqlType, columnSize, decDigits);
-
-            statement->BindParameter(paramIdx, param);
-        }
-        else
-            statement->UnbindParameter(paramIdx);
+        statement->BindParameter(paramIdx, ioType, bufferType, paramSqlType,
+            columnSize, decDigits, buffer, bufferLen, resLen);
 
         return statement->GetDiagnosticRecords().GetReturnCode();
     }
diff --git a/modules/platforms/cpp/odbc/src/query/batch_query.cpp b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
new file mode 100644
index 0000000000000..11ddd93e19302
--- /dev/null
+++ b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ignite/odbc/connection.h"
+#include "ignite/odbc/message.h"
+#include "ignite/odbc/query/batch_query.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        namespace query
+        {
+            BatchQuery::BatchQuery(diagnostic::Diagnosable& diag, Connection& connection,
+                const std::string& sql, const app::ParameterSet& params) :
+                Query(diag, BATCH),
+                connection(connection),
+                sql(sql),
+                params(params),
+                resultMeta(),
+                rowsAffected(0),
+                setsProcessed(0),
+                executed(false),
+                dataRetrieved(false)
+            {
+                // No-op.
+            }
+
+            BatchQuery::~BatchQuery()
+            {
+                // No-op.
+            }
+
+            SqlResult BatchQuery::Execute()
+            {
+                if (executed)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query cursor is in open state already.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                int32_t maxPageSize = connection.GetConfiguration().GetPageSize();
+                int32_t rowNum = params.GetParamSetSize();
+                SqlResult res;
+
+                int32_t processed = 0;
+
+                do {
+                    int32_t currentPageSize = std::min(maxPageSize, rowNum - processed);
+                    bool lastPage = currentPageSize == rowNum - processed;
+
+                    res = MakeRequestExecuteBatch(processed, processed + currentPageSize, lastPage);
+
+                    processed += currentPageSize;
+                } while (res == SQL_RESULT_SUCCESS && processed < rowNum);
+
+                params.SetParamsProcessed(static_cast<SqlUlen>(setsProcessed));
+
+                return res;
+            }
+
+            const meta::ColumnMetaVector& BatchQuery::GetMeta() const
+            {
+                return resultMeta;
+            }
+
+            SqlResult BatchQuery::FetchNextRow(app::ColumnBindingMap& columnBindings)
+            {
+                if (!executed)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query was not executed.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                if (dataRetrieved)
+                    return SQL_RESULT_NO_DATA;
+
+                app::ColumnBindingMap::iterator it = columnBindings.find(1);
+
+                if (it != columnBindings.end())
+                    it->second.PutInt64(rowsAffected);
+
+                dataRetrieved = true;
+
+                return SQL_RESULT_SUCCESS;
+            }
+
+            SqlResult BatchQuery::GetColumn(uint16_t columnIdx, app::ApplicationDataBuffer& buffer)
+            {
+                if (!executed)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query was not executed.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                if (dataRetrieved)
+                    return SQL_RESULT_NO_DATA;
+
+                if (columnIdx != 1)
+                {
+                    std::stringstream builder;
+                    builder << "Column with id " << columnIdx << " is not available in result set.";
+
+                    diag.AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR, builder.str());
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                buffer.PutInt64(rowsAffected);
+
+                return SQL_RESULT_SUCCESS;
+            }
+
+            SqlResult BatchQuery::Close()
+            {
+                return SQL_RESULT_SUCCESS;
+            }
+
+            bool BatchQuery::DataAvailable() const
+            {
+                return false;
+            }
+
+            int64_t BatchQuery::AffectedRows() const
+            {
+                return rowsAffected;
+            }
+
+            SqlResult BatchQuery::MakeRequestExecuteBatch(SqlUlen begin, SqlUlen end, bool last)
+            {
+                const std::string& schema = connection.GetCache();
+
+                QueryExecuteBatchtRequest req(schema, sql, params, begin, end, last);
+                QueryExecuteBatchResponse rsp;
+
+                try
+                {
+                    connection.SyncMessage(req, rsp);
+                }
+                catch (const IgniteError& err)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                if (rsp.GetStatus() != RESPONSE_STATUS_SUCCESS)
+                {
+                    LOG_MSG("Error: " << rsp.GetError());
+
+                    diag.AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR, rsp.GetError());
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                rowsAffected += rsp.GetAffectedRows();
+                LOG_MSG("rowsAffected: " << rowsAffected);
+
+                if (!rsp.GetErrorMessage().empty())
+                {
+                    LOG_MSG("Error: " << rsp.GetErrorMessage());
+
+                    setsProcessed += rsp.GetErrorSetIdx();
+                    LOG_MSG("setsProcessed: " << setsProcessed);
+
+                    diag.AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR, rsp.GetErrorMessage(),
+                        static_cast<int32_t>(setsProcessed), 0);
+
+                    return SQL_RESULT_SUCCESS_WITH_INFO;
+                }
+
+                setsProcessed += end - begin;
+                LOG_MSG("setsProcessed: " << setsProcessed);
+
+                return SQL_RESULT_SUCCESS;
+            }
+        }
+    }
+}
+
diff --git a/modules/platforms/cpp/odbc/src/query/data_query.cpp b/modules/platforms/cpp/odbc/src/query/data_query.cpp
index 5b1b758bd4281..f4402d41f7895 100644
--- a/modules/platforms/cpp/odbc/src/query/data_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/data_query.cpp
@@ -18,6 +18,7 @@
 #include "ignite/odbc/connection.h"
 #include "ignite/odbc/message.h"
 #include "ignite/odbc/query/data_query.h"
+#include "ignite/odbc/query/batch_query.h"
 
 namespace ignite
 {
@@ -25,9 +26,8 @@ namespace ignite
     {
         namespace query
         {
-            DataQuery::DataQuery(diagnostic::Diagnosable& diag,
-                Connection& connection, const std::string& sql,
-                const app::ParameterBindingMap& params) :
+            DataQuery::DataQuery(diagnostic::Diagnosable& diag, Connection& connection,
+                const std::string& sql, const app::ParameterSet& params) :
                 Query(diag, DATA),
                 connection(connection),
                 sql(sql),
@@ -38,9 +38,9 @@ namespace ignite
 
             DataQuery::~DataQuery()
             {
-                Close();
+                InternalClose();
             }
-            
+
             SqlResult DataQuery::Execute()
             {
                 if (cursor.get())
@@ -139,6 +139,11 @@ namespace ignite
             }
 
             SqlResult DataQuery::Close()
+            {
+                return InternalClose();
+            }
+
+            SqlResult DataQuery::InternalClose()
             {
                 if (!cursor.get())
                     return SQL_RESULT_SUCCESS;
diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp
index 9aca8c9164209..6154f9126568e 100644
--- a/modules/platforms/cpp/odbc/src/statement.cpp
+++ b/modules/platforms/cpp/odbc/src/statement.cpp
@@ -16,6 +16,7 @@
  */
 
 #include "ignite/odbc/system/odbc_constants.h"
+#include "ignite/odbc/query/batch_query.h"
 #include "ignite/odbc/query/data_query.h"
 #include "ignite/odbc/query/column_metadata_query.h"
 #include "ignite/odbc/query/table_metadata_query.h"
@@ -38,9 +39,8 @@ namespace ignite
             currentQuery(),
             rowsFetched(0),
             rowStatuses(0),
-            paramBindOffset(0),
             columnBindOffset(0),
-            currentParamIdx(0)
+            parameters()
         {
             // No-op.
         }
@@ -55,8 +55,6 @@ namespace ignite
             IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
 
             columnBindings[columnIdx] = buffer;
-
-            columnBindings[columnIdx].SetPtrToOffsetPtr(&columnBindOffset);
         }
 
         void Statement::UnbindColumn(uint16_t columnIdx)
@@ -108,25 +106,72 @@ namespace ignite
             return SQL_RESULT_SUCCESS;
         }
 
-        void Statement::BindParameter(uint16_t paramIdx, const app::Parameter& param)
+        void Statement::BindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType, int16_t paramSqlType,
+            SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen)
         {
-            IGNITE_ODBC_API_CALL(InternalBindParameter(paramIdx, param));
+            IGNITE_ODBC_API_CALL(InternalBindParameter(paramIdx, ioType, bufferType, paramSqlType,
+                columnSize, decDigits, buffer, bufferLen, resLen));
         }
 
-
-        SqlResult Statement::InternalBindParameter(uint16_t paramIdx, const app::Parameter& param)
+        SqlResult Statement::InternalBindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType,
+            int16_t paramSqlType, SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen)
         {
+            using namespace type_traits;
+            using app::ApplicationDataBuffer;
+            using app::Parameter;
+
             if (paramIdx == 0)
             {
-                AddStatusRecord(SQL_STATE_24000_INVALID_CURSOR_STATE,
-                    "The value specified for the argument ParameterNumber was less than 1.");
+                std::stringstream builder;
+                builder << "The value specified for the argument ParameterNumber was less than 1. [ParameterNumber=" << paramIdx << ']';
+
+                AddStatusRecord(SQL_STATE_24000_INVALID_CURSOR_STATE, builder.str());
+
+                return SQL_RESULT_ERROR;
+            }
+
+            if (ioType != SQL_PARAM_INPUT)
+            {
+                std::stringstream builder;
+                builder << "The value specified for the argument InputOutputType was not SQL_PARAM_INPUT. [ioType=" << ioType << ']';
+
+                AddStatusRecord(SQL_STATE_HY105_INVALID_PARAMETER_TYPE, builder.str());
+
+                return SQL_RESULT_ERROR;
+            }
+
+            if (!IsSqlTypeSupported(paramSqlType))
+            {
+                std::stringstream builder;
+                builder << "Data type is not supported. [typeId=" << paramSqlType << ']';
+
+                AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED, builder.str());
+
+                return SQL_RESULT_ERROR;
+            }
+
+            IgniteSqlType driverType = ToDriverType(bufferType);
+
+            if (driverType == IGNITE_ODBC_C_TYPE_UNSUPPORTED)
+            {
+                std::stringstream builder;
+                builder << "The argument TargetType was not a valid data type. [TargetType=" << bufferType << ']';
+
+                AddStatusRecord(SQL_STATE_HY003_INVALID_APPLICATION_BUFFER_TYPE, builder.str());
 
                 return SQL_RESULT_ERROR;
             }
 
-            paramBindings[paramIdx] = param;
+            if (buffer)
+            {
+                ApplicationDataBuffer dataBuffer(driverType, buffer, bufferLen, resLen);
+
+                Parameter param(dataBuffer, paramSqlType, columnSize, decDigits);
 
-            paramBindings[paramIdx].GetBuffer().SetPtrToOffsetPtr(&paramBindOffset);
+                parameters.BindParameter(paramIdx, param);
+            }
+            else
+                parameters.UnbindParameter(paramIdx);
 
             return SQL_RESULT_SUCCESS;
         }
@@ -135,14 +180,14 @@ namespace ignite
         {
             IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
 
-            paramBindings.erase(paramIdx);
+            parameters.UnbindParameter(paramIdx);
         }
 
         void Statement::UnbindAllParameters()
         {
             IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
 
-            paramBindings.clear();
+            parameters.UnbindAll();
         }
 
         void Statement::SetAttribute(int attr, void* value, SQLINTEGER valueLen)
@@ -156,7 +201,7 @@ namespace ignite
             {
                 case SQL_ATTR_ROW_ARRAY_SIZE:
                 {
-                    SQLULEN val = reinterpret_cast<SQLULEN>(value);
+                    SqlUlen val = reinterpret_cast<SqlUlen>(value);
 
                     LOG_MSG("SQL_ATTR_ROW_ARRAY_SIZE: %d\n", val);
 
@@ -199,6 +244,20 @@ namespace ignite
                     break;
                 }
 
+                case SQL_ATTR_PARAMSET_SIZE:
+                {
+                    parameters.SetParamSetSize(reinterpret_cast<SqlUlen>(value));
+
+                    break;
+                }
+
+                case SQL_ATTR_PARAMS_PROCESSED_PTR:
+                {
+                    parameters.SetParamsProcessedPtr(reinterpret_cast<SqlUlen*>(value));
+
+                    break;
+                }
+
                 default:
                 {
                     AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED,
@@ -236,6 +295,9 @@ namespace ignite
 
                     *val = static_cast<SQLPOINTER>(this);
 
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
+
                     break;
                 }
 
@@ -245,14 +307,20 @@ namespace ignite
 
                     *val = static_cast<SQLINTEGER>(1);
 
+                    if (valueLen)
+                        *valueLen = SQL_IS_INTEGER;
+
                     break;
                 }
 
                 case SQL_ATTR_ROWS_FETCHED_PTR:
                 {
-                    SQLULEN** val = reinterpret_cast<SQLULEN**>(buf);
+                    SqlUlen** val = reinterpret_cast<SqlUlen**>(buf);
+
+                    *val = reinterpret_cast<SqlUlen*>(GetRowsFetchedPtr());
 
-                    *val = reinterpret_cast<SQLULEN*>(GetRowsFetchedPtr());
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
 
                     break;
                 }
@@ -263,6 +331,9 @@ namespace ignite
 
                     *val = reinterpret_cast<SQLUSMALLINT*>(GetRowStatusesPtr());
 
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
+
                     break;
                 }
 
@@ -270,16 +341,46 @@ namespace ignite
                 {
                     SQLULEN** val = reinterpret_cast<SQLULEN**>(buf);
 
-                    *val = reinterpret_cast<SQLULEN*>(GetParamBindOffsetPtr());
+                    *val = reinterpret_cast<SQLULEN*>(parameters.GetParamBindOffsetPtr());
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
 
                     break;
                 }
 
                 case SQL_ATTR_ROW_BIND_OFFSET_PTR:
                 {
-                    SQLULEN** val = reinterpret_cast<SQLULEN**>(buf);
+                    SqlUlen** val = reinterpret_cast<SqlUlen**>(buf);
+
+                    *val = reinterpret_cast<SqlUlen*>(GetColumnBindOffsetPtr());
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
+
+                    break;
+                }
+
+                case SQL_ATTR_PARAMSET_SIZE:
+                {
+                    SqlUlen* val = reinterpret_cast<SqlUlen*>(buf);
+
+                    *val = static_cast<SqlUlen>(parameters.GetParamSetSize());
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_UINTEGER;
+
+                    break;
+                }
+
+                case SQL_ATTR_PARAMS_PROCESSED_PTR:
+                {
+                    SqlUlen** val = reinterpret_cast<SqlUlen**>(buf);
+
+                    *val = parameters.GetParamsProcessedPtr();
 
-                    *val = reinterpret_cast<SQLULEN*>(GetColumnBindOffsetPtr());
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
 
                     break;
                 }
@@ -317,7 +418,7 @@ namespace ignite
                 return SQL_RESULT_SUCCESS;
             }
 
-            if (paramTypes.empty())
+            if (!parameters.IsMetadataSet())
             {
                 SqlResult res = UpdateParamsMeta();
 
@@ -325,7 +426,7 @@ namespace ignite
                     return res;
             }
 
-            paramNum = static_cast<uint16_t>(paramTypes.size());
+            paramNum = parameters.GetExpectedParamNum();
 
             return SQL_RESULT_SUCCESS;
         }
@@ -334,12 +435,7 @@ namespace ignite
         {
             IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
 
-            paramBindOffset = ptr;
-        }
-
-        int* Statement::GetParamBindOffsetPtr()
-        {
-            return paramBindOffset;
+            parameters.SetParamBindOffsetPtr(ptr);
         }
 
         void Statement::GetColumnData(uint16_t columnIdx, app::ApplicationDataBuffer& buffer)
@@ -373,10 +469,10 @@ namespace ignite
             if (currentQuery.get())
                 currentQuery->Close();
 
-            currentQuery.reset(new query::DataQuery(*this, connection, query, paramBindings));
-
             // Resetting parameters types as we are changing the query.
-            paramTypes.clear();
+            parameters.Prepare();
+
+            currentQuery.reset(new query::DataQuery(*this, connection, query, parameters));
 
             return SQL_RESULT_SUCCESS;
         }
@@ -410,20 +506,31 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            bool paramDataReady = true;
-
-            app::ParameterBindingMap::iterator it;
-            for (it = paramBindings.begin(); it != paramBindings.end(); ++it)
+            if (parameters.GetParamSetSize() > 1 && currentQuery->GetType() == query::Query::DATA)
             {
-                app::Parameter& param = it->second;
+                query::DataQuery& qry = static_cast<query::DataQuery&>(*currentQuery);
 
-                param.ResetStoredData();
+                currentQuery.reset(new query::BatchQuery(*this, connection, qry.GetSql(), parameters));
+            }
+            else if (parameters.GetParamSetSize() == 1 && currentQuery->GetType() == query::Query::BATCH)
+            {
+                query::BatchQuery& qry = static_cast<query::BatchQuery&>(*currentQuery);
 
-                paramDataReady &= param.IsDataReady();
+                currentQuery.reset(new query::DataQuery(*this, connection, qry.GetSql(), parameters));
             }
 
-            if (!paramDataReady)
+            if (parameters.IsDataAtExecNeeded())
+            {
+                if (currentQuery->GetType() == query::Query::BATCH)
+                {
+                    AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED,
+                        "Data-at-execution is not supported together with batching.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
                 return SQL_RESULT_NEED_DATA;
+            }
 
             return currentQuery->Execute();
         }
@@ -549,8 +656,10 @@ namespace ignite
         {
             if (!type_traits::IsSqlTypeSupported(sqlType))
             {
-                AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED,
-                    "Data type is not supported.");
+                std::stringstream builder;
+                builder << "Data type is not supported. [typeId=" << sqlType << ']';
+
+                AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED, builder.str());
 
                 return SQL_RESULT_ERROR;
             }
@@ -572,7 +681,7 @@ namespace ignite
         {
             if (!currentQuery.get())
                 return SQL_RESULT_SUCCESS;
-            
+
             SqlResult result = currentQuery->Close();
 
             if (result == SQL_RESULT_SUCCESS)
@@ -599,6 +708,12 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
+            if (columnBindOffset)
+            {
+                for (app::ColumnBindingMap::iterator it = columnBindings.begin(); it != columnBindings.end(); ++it)
+                    it->second.SetByteOffset(*columnBindOffset);
+            }
+
             SqlResult res = currentQuery->FetchNextRow(columnBindings);
 
             if (res == SQL_RESULT_SUCCESS)
@@ -767,35 +882,24 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            app::ParameterBindingMap::iterator it;
+            app::Parameter *selected = parameters.GetSelectedParameter();
 
-            if (currentParamIdx)
+            if (selected && !selected->IsDataReady())
             {
-                it = paramBindings.find(currentParamIdx);
+                AddStatusRecord(SQL_STATE_22026_DATA_LENGTH_MISMATCH,
+                    "Less data was sent for a parameter than was specified with "
+                    "the StrLen_or_IndPtr argument in SQLBindParameter.");
 
-                if (it != paramBindings.end() && !it->second.IsDataReady())
-                {
-                    AddStatusRecord(SQL_STATE_22026_DATA_LENGTH_MISMATCH,
-                        "Less data was sent for a parameter than was specified with "
-                        "the StrLen_or_IndPtr argument in SQLBindParameter.");
-
-                    return SQL_RESULT_ERROR;
-                }
+                return SQL_RESULT_ERROR;
             }
 
-            for (it = paramBindings.begin(); it != paramBindings.end(); ++it)
-            {
-                uint16_t paramIdx = it->first;
-                app::Parameter& param = it->second;
-
-                if (!param.IsDataReady())
-                {
-                    *paramPtr = param.GetBuffer().GetData();
+            selected = parameters.SelectNextParameter();
 
-                    currentParamIdx = paramIdx;
+            if (selected)
+            {
+                *paramPtr = selected->GetBuffer().GetData();
 
-                    return SQL_RESULT_NEED_DATA;
-                }
+                return SQL_RESULT_NEED_DATA;
             }
 
             SqlResult res = currentQuery->Execute();
@@ -822,7 +926,7 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            if (currentParamIdx == 0)
+            if (!parameters.IsParameterSelected())
             {
                 AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR,
                     "Parameter is not selected with the SQLParamData.");
@@ -830,9 +934,9 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            app::ParameterBindingMap::iterator it = paramBindings.find(currentParamIdx);
+            app::Parameter* param = parameters.GetSelectedParameter();
 
-            if (it == paramBindings.end())
+            if (!param)
             {
                 AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR,
                     "Selected parameter has been unbound.");
@@ -840,9 +944,7 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            app::Parameter& param = it->second;
-
-            param.PutData(data, len);
+            param->PutData(data, len);
 
             return SQL_RESULT_SUCCESS;
         }
@@ -872,10 +974,7 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            int8_t type = 0;
-
-            if (paramNum > 0 && static_cast<size_t>(paramNum) <= paramTypes.size())
-                type = paramTypes[paramNum - 1];
+            int8_t type = parameters.GetParamType(paramNum, 0);
 
             LOG_MSG("Type: %d\n", type);
 
@@ -886,10 +985,7 @@ namespace ignite
                 if (res != SQL_RESULT_SUCCESS)
                     return res;
 
-                if (paramNum < 1 || static_cast<size_t>(paramNum) > paramTypes.size())
-                    type = impl::binary::IGNITE_HDR_NULL;
-                else
-                    type = paramTypes[paramNum - 1];
+                type = parameters.GetParamType(paramNum, impl::binary::IGNITE_HDR_NULL);
             }
 
             if (dataType)
@@ -942,10 +1038,12 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            paramTypes = rsp.GetTypeIds();
+            parameters.UpdateParamsTypes(rsp.GetTypeIds());
 
-            for (size_t i = 0; i < paramTypes.size(); ++i)
-                LOG_MSG("[%zu] Parameter type: %u\n", i, paramTypes[i]);
+            for (size_t i = 0; i < rsp.GetTypeIds().size(); ++i)
+            {
+                LOG_MSG("[" << i << "] Parameter type: " << rsp.GetTypeIds()[i]);
+            }
 
             return SQL_RESULT_SUCCESS;
         }

From 7fbaecc67f1b204162bda4595d6c118ddd45f963 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 16 Jun 2017 20:01:49 +0300
Subject: [PATCH 211/516] IGNITE-5527: Prevent starvation in stripe pool on
 unstable topology.

---
 .../processors/cache/GridDeferredAckMessageSender.java         | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java
index 7145dc2cd7fee..967404a1ccf3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java
@@ -173,7 +173,8 @@ private DeferredAckMessageBuffer(UUID nodeId) {
          * @return {@code True} if request was handled, {@code false} if this buffer is filled and cannot be used.
          */
         public boolean add(GridCacheVersion ver) {
-            readLock().lock();
+            if(!readLock().tryLock())
+                return false; // Here, writeLock is help by another thread and guard is already true.
 
             boolean snd = false;
 

From f81964f59b0ea5b8dfdc8eb2acc34d2a5b8fee07 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 10 Jan 2017 16:59:17 +0300
Subject: [PATCH 212/516] Do not evict removed entries, otherwise removes can
 be lost.

(cherry picked from commit 55ac6e7)
---
 .../internal/processors/cache/GridCacheMapEntry.java     | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 58b4ae3130c9d..ea01bca4bd50b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -29,7 +29,6 @@
 import javax.cache.expiry.ExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.EntryProcessorResult;
-
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -4300,6 +4299,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                     if (obsoleteVersionExtras() != null)
                         return true;
 
+                    // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                    if (cctx.deferredDelete() && deletedUnlocked())
+                        return false;
+
                     CacheObject prev = saveOldValueUnlocked(false);
 
                     if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {
@@ -4358,6 +4361,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                             // Version has changed since entry passed the filter. Do it again.
                             continue;
 
+                        // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                        if (cctx.deferredDelete() && deletedUnlocked())
+                            return false;
+
                         CacheObject prevVal = saveValueForIndexUnlocked();
 
                         if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {

From 38f0ea80f3d95be16b38b621b3bcc2910c463997 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 10 Jan 2017 16:59:17 +0300
Subject: [PATCH 213/516] Do not evict removed entries, otherwise removes can
 be lost.

(cherry picked from commit 55ac6e7)
---
 .../internal/processors/cache/GridCacheMapEntry.java     | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index d6d81de9d19a1..bf5652fd9ce60 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -29,7 +29,6 @@
 import javax.cache.expiry.ExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.EntryProcessorResult;
-
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -4292,6 +4291,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                     if (obsoleteVersionExtras() != null)
                         return true;
 
+                    // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                    if (cctx.deferredDelete() && deletedUnlocked())
+                        return false;
+
                     CacheObject prev = saveOldValueUnlocked(false);
 
                     if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {
@@ -4350,6 +4353,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                             // Version has changed since entry passed the filter. Do it again.
                             continue;
 
+                        // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                        if (cctx.deferredDelete() && deletedUnlocked())
+                            return false;
+
                         CacheObject prevVal = saveValueForIndexUnlocked();
 
                         if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {

From 5dd74ff635de50ff9561ccdb51bdeb620f60c3db Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 10 Jan 2017 16:59:17 +0300
Subject: [PATCH 214/516] Do not evict removed entries, otherwise removes can
 be lost.

(cherry picked from commit 55ac6e7)
---
 .../internal/processors/cache/GridCacheMapEntry.java     | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 58b4ae3130c9d..ea01bca4bd50b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -29,7 +29,6 @@
 import javax.cache.expiry.ExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.EntryProcessorResult;
-
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -4300,6 +4299,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                     if (obsoleteVersionExtras() != null)
                         return true;
 
+                    // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                    if (cctx.deferredDelete() && deletedUnlocked())
+                        return false;
+
                     CacheObject prev = saveOldValueUnlocked(false);
 
                     if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {
@@ -4358,6 +4361,10 @@ private synchronized <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                             // Version has changed since entry passed the filter. Do it again.
                             continue;
 
+                        // TODO GG-11241: need keep removed entries in heap map, otherwise removes can be lost.
+                        if (cctx.deferredDelete() && deletedUnlocked())
+                            return false;
+
                         CacheObject prevVal = saveValueForIndexUnlocked();
 
                         if (!hasReaders() && markObsolete0(obsoleteVer, false, null)) {

From c802b478ef47271f5b8864e0b0ae29925107e75f Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 21 Jun 2017 18:52:17 +0300
Subject: [PATCH 215/516] Compilation is fixed

---
 .../cache/store/cassandra/datasource/DataSource.java       | 3 ---
 .../service/GridServiceProcessorMultiNodeSelfTest.java     | 7 +++++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
index d07a183ade4eb..abf22a6d45832 100644
--- a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
+++ b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
@@ -56,9 +56,6 @@ public class DataSource implements Externalizable {
     /** */
     private static final long serialVersionUID = 0L;
 
-    /** Default expiration timeout for Cassandra driver session. */
-    public static final long DFLT_SESSION_EXPIRATION_TIMEOUT = 300000; // 5 minutes.
-
     /**
      * Null object, used as a replacement for those Cassandra connection options which
      * don't support serialization (RetryPolicy, LoadBalancingPolicy and etc).
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index 5728bffa102ad..2958b7cb248c8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -163,6 +163,8 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
 
             DummyService.exeLatch(name, latch);
 
+            int clients = 2;
+
             startExtraNodes(servers, clients);
 
             try {
@@ -170,6 +172,11 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
 
                 waitForDeployment(name, servers);
 
+                // Since we start extra nodes, there may be extra start and cancel events,
+                // so we check only the difference between start and cancel and
+                // not start and cancel events individually.
+                assertEquals(name, nodeCount() + servers,  DummyService.started(name) - DummyService.cancelled(name));
+
                 checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
             }
             finally {

From 5fb5c7e3b54ae4efb7a6a1832ba647677d93e0cd Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 22 Jun 2017 09:43:03 +0300
Subject: [PATCH 216/516] IGNITE-5399 Manual cache rebalancing feature is
 broken

---
 .../java/org/apache/ignite/IgniteCache.java   |   3 +-
 .../GridCachePartitionExchangeManager.java    |   6 +-
 .../processors/cache/GridCachePreloader.java  |   4 +-
 .../cache/GridCachePreloaderAdapter.java      |   4 +-
 .../preloader/GridDhtPartitionDemander.java   |  17 +-
 .../GridDhtPartitionsExchangeFuture.java      |   7 +-
 .../dht/preloader/GridDhtPreloader.java       |   2 +-
 .../CacheManualRebalancingTest.java           | 178 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |   4 +-
 9 files changed, 201 insertions(+), 24 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
index d7bccf58941b1..ef2c4b6a00166 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
@@ -872,7 +872,8 @@ public <T> Map<K, EntryProcessorResult<T>> invokeAll(Set<? extends K> keys,
      * See {@link CacheConfiguration#getRebalanceDelay()} for more information on how to configure
      * rebalance re-partition delay.
      * <p>
-     * @return Future that will be completed when rebalancing is finished.
+     * @return Future that will be completed when rebalancing is finished. Future.get() returns true
+     *      when rebalance was successfully finished.
      */
     public IgniteFuture<?> rebalance();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index c175c43ae2f2a..07805f3f1b50b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -80,6 +80,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridListSet;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.IgnitePair;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -724,7 +725,7 @@ public void forceDummyExchange(boolean reassign,
      * @param exchFut Exchange future.
      */
     public IgniteInternalFuture<Boolean> forceRebalance(GridDhtPartitionsExchangeFuture exchFut) {
-        GridFutureAdapter<Boolean> fut = new GridFutureAdapter<>();
+        GridCompoundFuture<Boolean, Boolean> fut = new GridCompoundFuture<>(CU.boolReducer());
 
         exchWorker.addFuture(
             new GridDhtPartitionsExchangeFuture(cctx, exchFut.discoveryEvent(), exchFut.exchangeId(), fut));
@@ -1830,6 +1831,9 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                             }
                         }
 
+                        if (exchFut.forcedRebalanceFuture() != null)
+                            exchFut.forcedRebalanceFuture().markInitialized();
+
                         if (assignsCancelled) { // Pending exchange.
                             U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
                                 "[top=" + exchFut.topologyVersion() + ", evt=" + exchFut.discoveryEvent().name() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index 0c2869101aa79..110a53b3d2154 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -28,7 +28,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
@@ -92,7 +92,7 @@ public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
         boolean forcePreload,
         int cnt,
         Runnable next,
-        @Nullable GridFutureAdapter<Boolean> forcedRebFut);
+        @Nullable GridCompoundFuture<Boolean, Boolean> forcedRebFut);
 
     /**
      * @param p Preload predicate.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index 8ae67215dabc9..fe4859b2dc4b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -30,8 +30,8 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
@@ -167,7 +167,7 @@ public GridCachePreloaderAdapter(GridCacheContext<?, ?> cctx) {
 
     /** {@inheritDoc} */
     @Override public Runnable addAssignments(GridDhtPreloaderAssignments assignments, boolean forcePreload,
-        int cnt, Runnable next, @Nullable GridFutureAdapter<Boolean> forcedRebFut) {
+        int cnt, Runnable next, @Nullable GridCompoundFuture<Boolean, Boolean> forcedRebFut) {
         return null;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 9ece00cd01049..5c6eb2b9a53d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -54,6 +54,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -301,7 +302,7 @@ Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
         boolean force,
         int cnt,
         final Runnable next,
-        @Nullable final GridFutureAdapter<Boolean> forcedRebFut) {
+        @Nullable final GridCompoundFuture<Boolean, Boolean> forcedRebFut) {
         if (log.isDebugEnabled())
             log.debug("Adding partition assignments: " + assigns);
 
@@ -324,18 +325,8 @@ Runnable addAssignments(final GridDhtPreloaderAssignments assigns,
                 });
             }
 
-            if (forcedRebFut != null) {
-                fut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
-                    @Override public void apply(IgniteInternalFuture<Boolean> future) {
-                        try {
-                            forcedRebFut.onDone(future.get());
-                        }
-                        catch (Exception e) {
-                            forcedRebFut.onDone(e);
-                        }
-                    }
-                });
-            }
+            if (forcedRebFut != null)
+                forcedRebFut.add(fut);
 
             rebalanceFut = fut;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 2245d17e83523..d3e3701936786 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -62,6 +62,7 @@
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -198,7 +199,7 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     private boolean centralizedAff;
 
     /** Forced Rebalance future. */
-    private GridFutureAdapter<Boolean> forcedRebFut;
+    private GridCompoundFuture<Boolean, Boolean> forcedRebFut;
 
     /**
      * Dummy future created to trigger reassignments if partition
@@ -236,7 +237,7 @@ public GridDhtPartitionsExchangeFuture(
      * @param forcedRebFut Forced Rebalance future.
      */
     public GridDhtPartitionsExchangeFuture(GridCacheSharedContext cctx, DiscoveryEvent discoEvt,
-        GridDhtPartitionExchangeId exchId, GridFutureAdapter<Boolean> forcedRebFut) {
+        GridDhtPartitionExchangeId exchId, GridCompoundFuture<Boolean, Boolean> forcedRebFut) {
         dummy = false;
         forcePreload = true;
 
@@ -417,7 +418,7 @@ public GridDhtPartitionExchangeId exchangeId() {
     /**
      * @return Forced Rebalance future.
      */
-    @Nullable public GridFutureAdapter<Boolean> forcedRebalanceFuture() {
+    @Nullable public GridCompoundFuture<Boolean, Boolean> forcedRebalanceFuture() {
         return forcedRebFut;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 4aff4d5a71f42..2efaed859b714 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -413,7 +413,7 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
 
     /** {@inheritDoc} */
     @Override public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
-        boolean forcePreload, int cnt, Runnable next,  @Nullable GridFutureAdapter<Boolean> forcedRebFut) {
+        boolean forcePreload, int cnt, Runnable next,  @Nullable GridCompoundFuture<Boolean, Boolean> forcedRebFut) {
         return demander.addAssignments(assignments, forcePreload, cnt, next, forcedRebFut);
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java
new file mode 100644
index 0000000000000..363e07b890711
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/** */
+public class CacheManualRebalancingTest extends GridCommonAbstractTest {
+    /** */
+    private static final String MYCACHE = "mycache";
+
+    /** */
+    public static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    public static final int NODES_CNT = 2;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setCacheConfiguration(cacheConfiguration());
+
+        return cfg;
+    }
+
+    /**
+     * @return Cache configuration.
+     */
+    private static CacheConfiguration cacheConfiguration() {
+        return new CacheConfiguration(MYCACHE)
+            .setAtomicityMode(ATOMIC)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setWriteSynchronizationMode(FULL_SYNC)
+            .setRebalanceMode(ASYNC)
+            .setRebalanceDelay(-1)
+            .setBackups(1)
+            .setCopyOnRead(true)
+            .setReadFromBackup(true);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrids(NODES_CNT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 400_000;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRebalance() throws Exception {
+        // Fill cache with large dataset to make rebalancing slow.
+        try (IgniteDataStreamer<Object, Object> streamer = grid(0).dataStreamer(MYCACHE)) {
+            for (int i = 0; i < 100_000; i++)
+                streamer.addData(i, i);
+        }
+
+        // Start new node.
+        final IgniteEx newNode = startGrid(NODES_CNT);
+
+        int newNodeCacheSize;
+
+        // Start manual rebalancing.
+        IgniteCompute compute = newNode.compute().withAsync();
+
+        compute.broadcast(new MyCallable());
+
+        final ComputeTaskFuture<Object> rebalanceTaskFuture = compute.future();
+
+        boolean rebalanceFinished = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return rebalanceTaskFuture.isDone();
+            }
+        }, 10_000);
+
+        assertTrue(rebalanceFinished);
+
+        assertTrue(newNode.context().cache().cache(MYCACHE).context().preloader().rebalanceFuture().isDone());
+
+        newNodeCacheSize = newNode.cache(MYCACHE).localSize(CachePeekMode.ALL);
+
+        System.out.println("New node cache local size: " + newNodeCacheSize);
+
+        assertTrue(newNodeCacheSize > 0);
+
+    }
+
+    /** */
+    public static class MyCallable implements IgniteRunnable {
+        /** */
+        @IgniteInstanceResource
+        Ignite localNode;
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            IgniteLogger log = localNode.log();
+
+            log.info("Start local rebalancing caches");
+
+            for (String cacheName : localNode.cacheNames()) {
+                IgniteCache<?, ?> cache = localNode.cache(cacheName);
+
+                assertNotNull(cache);
+
+                boolean finished;
+                
+                log.info("Start rebalancing cache: " + cacheName + ", size: " + cache.localSize());
+
+                do {
+                    IgniteFuture<?> rebalance = cache.rebalance();
+
+                    log.info("Wait rebalancing cache: " + cacheName + " - " + rebalance);
+
+                    finished = (Boolean)rebalance.get();
+
+                    log.info("Rebalancing cache: " + cacheName + " - " + rebalance);
+
+                    if (finished) {
+                        log.info("Finished rebalancing cache: " + cacheName + ", size: " +
+                            cache.localSize(CachePeekMode.PRIMARY) + cache.localSize(CachePeekMode.BACKUP));
+                    } else
+                        log.info("Rescheduled rebalancing cache: " + cacheName + ", size: " + cache.localSize());
+                }
+                while (!finished);
+            }
+
+            log.info("Finished local rebalancing caches");
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 58a26e76ddd2f..464ded85c8587 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -31,8 +31,9 @@
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentFairAffinityTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest;
 import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest;
 
 /**
@@ -66,6 +67,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheOffHeapCleanupTest.class);
 
         suite.addTestSuite(CacheRebalancingSelfTest.class);
+        suite.addTestSuite(CacheManualRebalancingTest.class);
 
         return suite;
     }

From 01d41b72ecc3e81dfc8966cc0e395c247037241c Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Wed, 21 Jun 2017 13:48:15 +0300
Subject: [PATCH 217/516] GG-12256 H2Indexes are not deleted if key class
 implements Externalizable

---
 .../processors/query/GridQueryProcessor.java  |  26 ++-
 .../cache/SqlQueryAfterCacheClearedTest.java  | 163 ++++++++++++++++++
 2 files changed, 187 insertions(+), 2 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 0bde688ded248..c79a65a6f1347 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -268,6 +268,17 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                         else
                             desc.keyClass(Object.class);
                     }
+                    else if (binaryEnabled) {
+                        if (!valMustDeserialize && !SQL_TYPES.contains(valCls))
+                            desc.valueClass(Object.class);
+                        else
+                            desc.valueClass(valCls);
+
+                        if (!keyMustDeserialize && !SQL_TYPES.contains(keyCls))
+                            desc.keyClass(Object.class);
+                        else
+                            desc.keyClass(keyCls);
+                    }
                     else {
                         if (keyCls == null)
                             throw new IgniteCheckedException("Failed to find key class in the node classpath " +
@@ -298,7 +309,7 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     TypeId typeId;
                     TypeId altTypeId = null;
 
-                    if (valCls == null || (binaryEnabled && !keyOrValMustDeserialize)) {
+                    if (valCls == null || (binaryEnabled && (!valMustDeserialize))) {
                         processBinaryMeta(qryEntity, desc);
 
                         typeId = new TypeId(ccfg.getName(), ctx.cacheObjects().typeId(qryEntity.getValueType()));
@@ -385,6 +396,17 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                         else
                             desc.keyClass(Object.class);
                     }
+                    else if (binaryEnabled) {
+                        if (!valMustDeserialize && !SQL_TYPES.contains(valCls))
+                            desc.valueClass(Object.class);
+                        else
+                            desc.valueClass(valCls);
+
+                        if (!keyMustDeserialize && !SQL_TYPES.contains(keyCls))
+                            desc.keyClass(Object.class);
+                        else
+                            desc.keyClass(keyCls);
+                    }
                     else {
                         desc.valueClass(valCls);
                         desc.keyClass(keyCls);
@@ -407,7 +429,7 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     TypeId typeId;
                     TypeId altTypeId = null;
 
-                    if (valCls == null || (binaryEnabled && !keyOrValMustDeserialize)) {
+                    if (valCls == null || (binaryEnabled && !valMustDeserialize)) {
                         processBinaryMeta(meta, desc);
 
                         typeId = new TypeId(ccfg.getName(), ctx.cacheObjects().typeId(meta.getValueType()));
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java
new file mode 100644
index 0000000000000..fa2d9ca341030
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.affinity.AffinityKey;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Test;
+
+import java.util.LinkedHashMap;
+
+import static java.util.Collections.singletonList;
+import static org.apache.ignite.cache.CacheMemoryMode.OFFHEAP_TIERED;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+public class SqlQueryAfterCacheClearedTest extends GridCommonAbstractTest {
+
+
+    /** */
+    public static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    private final static String CACHE_NAME = "propertyCache";
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrids(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        return new IgniteConfiguration()
+            .setGridName(gridName)
+            .setPeerClassLoadingEnabled(false)
+            .setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER))
+            .setCacheConfiguration(cacheCfg());
+    }
+
+    /** */
+    private static CacheConfiguration cacheCfg() {
+        final CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME)
+        .setCacheMode(PARTITIONED)
+        .setMemoryMode(OFFHEAP_TIERED)
+        .setQueryEntities(singletonList(createQueryEntityConfig()));
+        return ccfg;
+    }
+
+    @Test
+    public void testQueryCacheWasCleared() throws InterruptedException {
+        IgniteCache<PropertyAffinityKey, Property> cache = grid(0).cache(CACHE_NAME);
+
+        Property property1 = new Property(1, 2);
+        Property property2 = new Property(2, 2);
+
+        cache.put(property1.getKey(), property1);
+        cache.put(property2.getKey(), property2);
+
+        assertEquals(cache.size(),2);
+        assertEquals(cache.query(selectAllQuery()).getAll().size(), 2);
+
+        cache.clear();
+
+        assertEquals(0, cache.size());
+        assertEquals(0, cache.query(selectAllQuery()).getAll().size());
+    }
+
+    @Test
+    public void testQueryEntriesWereRemoved() {
+        IgniteCache<PropertyAffinityKey, Property> cache = grid(0).cache(CACHE_NAME);
+
+        Property property1 = new Property(1, 2);
+        Property property2 = new Property(2, 2);
+
+        cache.put(property1.getKey(), property1);
+        cache.put(property2.getKey(), property2);
+
+        assertEquals(cache.size(),2);
+        assertEquals(cache.query(selectAllQuery()).getAll().size(), 2);
+
+        cache.remove(new PropertyAffinityKey(1, 2));
+        cache.remove(new PropertyAffinityKey(2, 2));
+
+        assertEquals(0, cache.size());
+        assertEquals(0, cache.query(selectAllQuery()).getAll().size());
+    }
+
+    @NotNull
+    private SqlQuery<PropertyAffinityKey, Property> selectAllQuery() {
+        return new SqlQuery<>(Property.class, "from Property");
+    }
+
+    private static QueryEntity createQueryEntityConfig() {
+        QueryEntity queryEntity = new QueryEntity();
+        queryEntity.setKeyType(PropertyAffinityKey.class.getName());
+        queryEntity.setValueType(Property.class.getName());
+        queryEntity.setFields(getMapOfFields());
+        return queryEntity;
+    }
+
+    @NotNull
+    private static LinkedHashMap<String, String> getMapOfFields() {
+        LinkedHashMap<String, String> mapOfFields = new LinkedHashMap<>();
+        mapOfFields.put("id", Integer.class.getName());
+        mapOfFields.put("region", Integer.class.getName());
+        mapOfFields.put("key", PropertyAffinityKey.class.getName());
+        return mapOfFields;
+    }
+
+    static class Property {
+        private final int id;
+        private final int region;
+        private final String someData = "Some attributes";
+
+        Property(int id, int region) {
+            this.id = id;
+            this.region = region;
+        }
+
+        public PropertyAffinityKey getKey() {
+            return new PropertyAffinityKey(id, region);
+        }
+
+        public int getId() {
+            return id;
+        }
+    }
+
+    static class PropertyAffinityKey extends AffinityKey<Integer> {
+        public PropertyAffinityKey(final int thirdPartyPropertyId, final int region) {
+            super(thirdPartyPropertyId, region);
+        }
+
+        public PropertyAffinityKey() {
+            // Required by Ignite
+        }
+    }
+}

From 5ac9afc719138e37a7d97d9d9db05243eee9a942 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 22 Jun 2017 12:36:14 +0300
Subject: [PATCH 218/516] IGNITE-5399 add test to testsuite

---
 .../ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java       | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java
index c4fcdacb603c8..2bcfd1dbec332 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.SqlQueryAfterCacheClearedTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFailoverAtomicPrimaryWriteOrderSelfTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFailoverTxReplicatedSelfTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFailoverTxSelfTest;
@@ -50,6 +51,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheContinuousQueryAsyncFailoverAtomicPrimaryWriteOrderSelfTest.class);
         suite.addTestSuite(CacheContinuousQueryAsyncFailoverTxReplicatedSelfTest.class);
         suite.addTestSuite(CacheContinuousQueryAsyncFailoverTxSelfTest.class);
+        suite.addTestSuite(SqlQueryAfterCacheClearedTest.class);
 
         return suite;
     }

From a935d40a80e2f928a84a145aba540a45b156687f Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 22 Jun 2017 15:10:32 +0300
Subject: [PATCH 219/516] GG-12256 Minor fixes

---
 .../processors/query/GridQueryProcessor.java  | 28 +------
 .../cache/SqlQueryAfterCacheClearedTest.java  | 75 +++++++++++--------
 2 files changed, 46 insertions(+), 57 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index c79a65a6f1347..7511a991472c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -256,19 +256,7 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
 
                     desc.tableName(qryEntity.getTableName());
 
-                    if (binaryEnabled && !keyOrValMustDeserialize) {
-                        // Safe to check null.
-                        if (SQL_TYPES.contains(valCls))
-                            desc.valueClass(valCls);
-                        else
-                            desc.valueClass(Object.class);
-
-                        if (SQL_TYPES.contains(keyCls))
-                            desc.keyClass(keyCls);
-                        else
-                            desc.keyClass(Object.class);
-                    }
-                    else if (binaryEnabled) {
+                    if (binaryEnabled) {
                         if (!valMustDeserialize && !SQL_TYPES.contains(valCls))
                             desc.valueClass(Object.class);
                         else
@@ -384,19 +372,7 @@ else if (binaryEnabled) {
 
                     desc.name(simpleValType);
 
-                    if (binaryEnabled && !keyOrValMustDeserialize) {
-                        // Safe to check null.
-                        if (SQL_TYPES.contains(valCls))
-                            desc.valueClass(valCls);
-                        else
-                            desc.valueClass(Object.class);
-
-                        if (SQL_TYPES.contains(keyCls))
-                            desc.keyClass(keyCls);
-                        else
-                            desc.keyClass(Object.class);
-                    }
-                    else if (binaryEnabled) {
+                    if (binaryEnabled) {
                         if (!valMustDeserialize && !SQL_TYPES.contains(valCls))
                             desc.valueClass(Object.class);
                         else
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java
index fa2d9ca341030..669161fb98ac5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlQueryAfterCacheClearedTest.java
@@ -27,7 +27,6 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.NotNull;
-import org.junit.Test;
 
 import java.util.LinkedHashMap;
 
@@ -35,13 +34,15 @@
 import static org.apache.ignite.cache.CacheMemoryMode.OFFHEAP_TIERED;
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 
+/**
+ *
+ */
 public class SqlQueryAfterCacheClearedTest extends GridCommonAbstractTest {
-
-
     /** */
     public static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
-    private final static String CACHE_NAME = "propertyCache";
+    /** */
+    private static final String CACHE_NAME = "propertyCache";
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
@@ -64,22 +65,21 @@ public class SqlQueryAfterCacheClearedTest extends GridCommonAbstractTest {
 
     /** */
     private static CacheConfiguration cacheCfg() {
-        final CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME)
+        return new CacheConfiguration(CACHE_NAME)
         .setCacheMode(PARTITIONED)
         .setMemoryMode(OFFHEAP_TIERED)
         .setQueryEntities(singletonList(createQueryEntityConfig()));
-        return ccfg;
     }
 
-    @Test
+    /** */
     public void testQueryCacheWasCleared() throws InterruptedException {
         IgniteCache<PropertyAffinityKey, Property> cache = grid(0).cache(CACHE_NAME);
 
-        Property property1 = new Property(1, 2);
-        Property property2 = new Property(2, 2);
+        Property prop1 = new Property(1, 2);
+        Property prop2 = new Property(2, 2);
 
-        cache.put(property1.getKey(), property1);
-        cache.put(property2.getKey(), property2);
+        cache.put(prop1.getKey(), prop1);
+        cache.put(prop2.getKey(), prop2);
 
         assertEquals(cache.size(),2);
         assertEquals(cache.query(selectAllQuery()).getAll().size(), 2);
@@ -90,15 +90,15 @@ public void testQueryCacheWasCleared() throws InterruptedException {
         assertEquals(0, cache.query(selectAllQuery()).getAll().size());
     }
 
-    @Test
+    /** */
     public void testQueryEntriesWereRemoved() {
         IgniteCache<PropertyAffinityKey, Property> cache = grid(0).cache(CACHE_NAME);
 
-        Property property1 = new Property(1, 2);
-        Property property2 = new Property(2, 2);
+        Property prop1 = new Property(1, 2);
+        Property prop2 = new Property(2, 2);
 
-        cache.put(property1.getKey(), property1);
-        cache.put(property2.getKey(), property2);
+        cache.put(prop1.getKey(), prop1);
+        cache.put(prop2.getKey(), prop2);
 
         assertEquals(cache.size(),2);
         assertEquals(cache.query(selectAllQuery()).getAll().size(), 2);
@@ -110,21 +110,22 @@ public void testQueryEntriesWereRemoved() {
         assertEquals(0, cache.query(selectAllQuery()).getAll().size());
     }
 
-    @NotNull
-    private SqlQuery<PropertyAffinityKey, Property> selectAllQuery() {
+    /** */
+    @NotNull private SqlQuery<PropertyAffinityKey, Property> selectAllQuery() {
         return new SqlQuery<>(Property.class, "from Property");
     }
 
+    /** */
     private static QueryEntity createQueryEntityConfig() {
-        QueryEntity queryEntity = new QueryEntity();
-        queryEntity.setKeyType(PropertyAffinityKey.class.getName());
-        queryEntity.setValueType(Property.class.getName());
-        queryEntity.setFields(getMapOfFields());
-        return queryEntity;
+        QueryEntity qryEntity = new QueryEntity();
+        qryEntity.setKeyType(PropertyAffinityKey.class.getName());
+        qryEntity.setValueType(Property.class.getName());
+        qryEntity.setFields(getMapOfFields());
+        return qryEntity;
     }
 
-    @NotNull
-    private static LinkedHashMap<String, String> getMapOfFields() {
+    /** */
+    @NotNull private static LinkedHashMap<String, String> getMapOfFields() {
         LinkedHashMap<String, String> mapOfFields = new LinkedHashMap<>();
         mapOfFields.put("id", Integer.class.getName());
         mapOfFields.put("region", Integer.class.getName());
@@ -132,32 +133,44 @@ private static LinkedHashMap<String, String> getMapOfFields() {
         return mapOfFields;
     }
 
-    static class Property {
+    /**
+     *
+     */
+    private static class Property {
+        /** Id. */
         private final int id;
+
+        /** Region. */
         private final int region;
-        private final String someData = "Some attributes";
 
+        /** */
         Property(int id, int region) {
             this.id = id;
             this.region = region;
         }
 
+        /** */
         public PropertyAffinityKey getKey() {
             return new PropertyAffinityKey(id, region);
         }
 
+        /** */
         public int getId() {
             return id;
         }
     }
 
-    static class PropertyAffinityKey extends AffinityKey<Integer> {
-        public PropertyAffinityKey(final int thirdPartyPropertyId, final int region) {
-            super(thirdPartyPropertyId, region);
+    /**
+     *
+     */
+    private static class PropertyAffinityKey extends AffinityKey<Integer> {
+        /** */
+        PropertyAffinityKey(final int thirdPartyPropId, final int region) {
+            super(thirdPartyPropId, region);
         }
 
+        /** */
         public PropertyAffinityKey() {
-            // Required by Ignite
         }
     }
 }

From 7e2468770a4eb47a4f61204d8c2000b6ab67c967 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Thu, 22 Jun 2017 16:13:01 +0300
Subject: [PATCH 220/516] IGNITE-GG-12197 Fixed "Ignore events for discarded
 update in CLOCK mode".

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../processors/cache/GridCacheMapEntry.java   |  10 +-
 .../CacheContinuousQueryHandler.java          | 128 +++++++-----
 .../CacheContinuousQueryManager.java          |   5 +-
 .../cache/version/GridCacheVersion.java       |   9 +
 ...tinuousQueryClockModeConflictSelfTest.java | 196 ++++++++++++++++++
 ...ntinuousQueryFailoverAbstractSelfTest.java |  14 +-
 ...CacheContinuousQueryOrderingEventTest.java | 131 ++++++++----
 .../IgniteCacheQuerySelfTestSuite3.java       |   2 +
 8 files changed, 392 insertions(+), 103 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryClockModeConflictSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index ea01bca4bd50b..e8c449994a253 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -2280,16 +2280,12 @@ else if (ttl != CU.TTL_ZERO)
                             if (updateCntr != null)
                                 updateCntr0 = updateCntr;
 
-                            cctx.continuousQueries().onEntryUpdated(
+                            cctx.continuousQueries().skipUpdateEvent(
+                                lsnrs,
                                 key,
-                                evtVal,
-                                prevVal,
-                                isInternal() || !context().userCache(),
                                 partition(),
-                                primary,
-                                false,
                                 updateCntr0,
-                                null,
+                                primary,
                                 topVer);
                         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 17f4308c0709b..db8575c5ff89e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -137,6 +137,9 @@ public class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler
     /** */
     private boolean locCache;
 
+    /** Local query. */
+    private transient boolean localQuery;
+
     /** */
     private transient boolean keepBinary;
 
@@ -242,6 +245,13 @@ public void localCache(boolean locCache) {
         this.locCache = locCache;
     }
 
+    /**
+     * @param loc Local query.
+     */
+    public void localQuery(boolean loc) {
+        this.localQuery = loc;
+    }
+
     /**
      * @param taskHash Task hash.
      */
@@ -334,13 +344,13 @@ public void keepBinary(boolean keepBinary) {
             }
         }
 
-        entryBufs = new ConcurrentHashMap<>();
-
-        backupQueue = new ConcurrentLinkedDeque8<>();
-
-        ackBuf = new AcknowledgeBuffer();
-
-        rcvs = new ConcurrentHashMap<>();
+        // Not need to support Fault Tolerance for local queries or local cache.
+        if (!isQueryOnlyLocal()) {
+            entryBufs = new ConcurrentHashMap<>();
+            backupQueue = new ConcurrentLinkedDeque8<>();
+            ackBuf = new AcknowledgeBuffer();
+            rcvs = new ConcurrentHashMap<>();
+        }
 
         this.nodeId = nodeId;
 
@@ -403,7 +413,7 @@ public void keepBinary(boolean keepBinary) {
                         primary,
                         evt,
                         recordIgniteEvt,
-                        fut);
+                        isQueryOnlyLocal() ? null : fut);
 
                     ctx.asyncCallbackPool().execute(clsr, evt.partitionId());
                 }
@@ -415,7 +425,7 @@ public void keepBinary(boolean keepBinary) {
                             + ", notify=" + notify + ']');
 
                     if (primary || skipPrimaryCheck) {
-                        if (fut == null)
+                        if (fut == null || isQueryOnlyLocal())
                             onEntryUpdate(evt, notify, loc, recordIgniteEvt);
                         else {
                             fut.addContinuousQueryClosure(new CI1<Boolean>() {
@@ -437,57 +447,66 @@ public void keepBinary(boolean keepBinary) {
             }
 
             @Override public void cleanupBackupQueue(Map<Integer, Long> updateCntrs) {
-                Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
+                if (!isQueryOnlyLocal()) {
+                    Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
 
-                if (backupQueue0 != null) {
-                    Iterator<CacheContinuousQueryEntry> it = backupQueue0.iterator();
+                    if (backupQueue0 != null) {
+                        Iterator<CacheContinuousQueryEntry> it = backupQueue0.iterator();
 
-                    while (it.hasNext()) {
-                        CacheContinuousQueryEntry backupEntry = it.next();
+                        while (it.hasNext()) {
+                            CacheContinuousQueryEntry backupEntry = it.next();
 
-                        Long updateCntr = updateCntrs.get(backupEntry.partition());
+                            Long updateCntr = updateCntrs.get(backupEntry.partition());
 
-                        if (updateCntr != null && backupEntry.updateCounter() <= updateCntr)
-                            it.remove();
+                            if (updateCntr != null && backupEntry.updateCounter() <= updateCntr)
+                                it.remove();
+                        }
                     }
                 }
             }
 
             @Override public void flushBackupQueue(GridKernalContext ctx, AffinityTopologyVersion topVer) {
-                Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
+                if (!isQueryOnlyLocal()) {
+                    Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
 
-                if (backupQueue0 == null)
-                    return;
+                    if (backupQueue0 == null)
+                        return;
 
-                try {
-                    ClusterNode nodeId0 = ctx.discovery().node(nodeId);
+                    try {
+                        ClusterNode nodeId0 = ctx.discovery().node(nodeId);
 
-                    if (nodeId0 != null) {
-                        GridCacheContext<K, V> cctx = cacheContext(ctx);
+                        if (nodeId0 != null) {
+                            GridCacheContext<K, V> cctx = cacheContext(ctx);
 
-                        for (CacheContinuousQueryEntry e : backupQueue0) {
-                            if (!e.isFiltered())
-                                prepareEntry(cctx, nodeId, e);
+                            for (CacheContinuousQueryEntry e : backupQueue0) {
+                                if (!e.isFiltered())
+                                    prepareEntry(cctx, nodeId, e);
+
+                                e.topologyVersion(topVer);
+                            }
 
-                            e.topologyVersion(topVer);
+                            ctx.continuous().addBackupNotification(nodeId, routineId, backupQueue0, topic);
                         }
+                        else
+                            // Node which start CQ leave topology. Not needed to put data to backup queue.
+                            backupQueue = null;
 
-                        ctx.continuous().addBackupNotification(nodeId, routineId, backupQueue0, topic);
+                        backupQueue0.clear();
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY),
+                            "Failed to send backup event notification to node: " + nodeId, e);
                     }
-                    else
-                        // Node which start CQ leave topology. Not needed to put data to backup queue.
-                        backupQueue = null;
-
-                    backupQueue0.clear();
-                }
-                catch (IgniteCheckedException e) {
-                    U.error(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY),
-                        "Failed to send backup event notification to node: " + nodeId, e);
                 }
+                else
+                    assert backupQueue == null; // For local CQ backup queue should be null.
             }
 
             @Override public void acknowledgeBackupOnTimeout(GridKernalContext ctx) {
-                sendBackupAcknowledge(ackBuf.acknowledgeOnTimeout(), routineId, ctx);
+                if (!localQuery)
+                    sendBackupAcknowledge(ackBuf.acknowledgeOnTimeout(), routineId, ctx);
+                else
+                    assert ackBuf == null; // For local CQ ack buffer should be null.
             }
 
             @Override public void skipUpdateEvent(CacheContinuousQueryEvent<K, V> evt,
@@ -502,14 +521,18 @@ public void keepBinary(boolean keepBinary) {
             }
 
             @Override public void onPartitionEvicted(int part) {
-                Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
+                if (!isQueryOnlyLocal()) {
+                    Collection<CacheContinuousQueryEntry> backupQueue0 = backupQueue;
 
-                if (backupQueue0 != null) {
-                    for (Iterator<CacheContinuousQueryEntry> it = backupQueue0.iterator(); it.hasNext(); ) {
-                        if (it.next().partition() == part)
-                            it.remove();
+                    if (backupQueue0 != null) {
+                        for (Iterator<CacheContinuousQueryEntry> it = backupQueue0.iterator(); it.hasNext(); ) {
+                            if (it.next().partition() == part)
+                                it.remove();
+                        }
                     }
                 }
+                else
+                    assert backupQueue == null; // For local CQ backup queue should be null.
             }
 
             @Override public boolean oldValueRequired() {
@@ -788,7 +811,11 @@ private void onEntryUpdate(CacheContinuousQueryEvent evt, boolean notify, boolea
             final CacheContinuousQueryEntry entry = evt.entry();
 
             if (loc) {
-                if (!locCache) {
+                if (isQueryOnlyLocal()) {
+                    if (!entry.isFiltered())
+                        locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
+                }
+                else {
                     Collection<CacheEntryEvent<? extends K, ? extends V>> evts = handleEvent(ctx, entry);
 
                     if (!evts.isEmpty())
@@ -797,10 +824,6 @@ private void onEntryUpdate(CacheContinuousQueryEvent evt, boolean notify, boolea
                     if (!internal && !skipPrimaryCheck)
                         sendBackupAcknowledge(ackBuf.onAcknowledged(entry), routineId, ctx);
                 }
-                else {
-                    if (!entry.isFiltered())
-                        locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
-                }
             }
             else {
                 if (!entry.isFiltered())
@@ -848,6 +871,13 @@ private void onEntryUpdate(CacheContinuousQueryEvent evt, boolean notify, boolea
         }
     }
 
+    /**
+     * @return {@code True} if query deployed only localy (local query or local cache), otherwise {@code false}.
+     */
+    private boolean isQueryOnlyLocal() {
+        return locCache || localQuery;
+    }
+
     /**
      * @return Task name.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 91c199184a2de..9a4ca90269727 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -641,6 +641,7 @@ private UUID executeQuery0(CacheEntryUpdatedListener locLsnr,
         hnd.internal(internal);
         hnd.keepBinary(keepBinary);
         hnd.localCache(cctx.isLocal());
+        hnd.localQuery(loc);
 
         IgnitePredicate<ClusterNode> pred = (loc || cctx.config().getCacheMode() == CacheMode.LOCAL) ?
             F.nodeForNodeId(cctx.localNodeId()) : cctx.config().getNodeFilter();
@@ -649,14 +650,14 @@ private UUID executeQuery0(CacheEntryUpdatedListener locLsnr,
 
         UUID id = cctx.kernalContext().continuous().startRoutine(
             hnd,
-            internal && loc,
+            loc,
             bufSize,
             timeInterval,
             autoUnsubscribe,
             pred).get();
 
         try {
-            if (hnd.isQuery() && cctx.userCache() && !onStart)
+            if (hnd.isQuery() && cctx.userCache() && !onStart && !loc)
                 hnd.waitTopologyFuture(cctx.kernalContext());
         }
         catch (IgniteCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersion.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersion.java
index 95aab7407116c..7796dd5f9cdbc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersion.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersion.java
@@ -190,6 +190,15 @@ public IgniteUuid asGridUuid() {
         return new IgniteUuid(new UUID(((long)topVer << 32) | nodeOrderDrId, globalTime), order);
     }
 
+    /**
+     * For testing purpose!!!
+     *
+     * @param globalTime Global time.
+     */
+    public void globalTime(long globalTime) {
+        this.globalTime = globalTime;
+    }
+
     /** {@inheritDoc} */
     @Override public void onAckReceived() {
         // No-op.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryClockModeConflictSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryClockModeConflictSelfTest.java
new file mode 100644
index 0000000000000..9de738346090c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryClockModeConflictSelfTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryListenerException;
+import javax.cache.event.CacheEntryUpdatedListener;
+import javax.cache.event.EventType;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicWriteOrderMode;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicFullUpdateRequest;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test coverages case when update discarded
+ */
+public class CacheContinuousQueryClockModeConflictSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    public static final int KEYS = 0;
+
+    /** */
+    public static final int ITERATION_CNT = 50;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+        cfg.setCommunicationSpi(new TestCommunicationSpi());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        // Node should start exactly in the same order.
+        startGrid(0);
+        startGrid(1);
+        startGrid(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testIgniteLocalContinuousQuery() throws Exception {
+        for (int i = 0; i < ITERATION_CNT; i++) {
+            String cacheName = "test-cache-" + i;
+
+            IgniteCache cache2 = ignite(2).createCache(cacheConfiguration(cacheName));
+
+            try {
+                IgniteCache cache0 = ignite(0).cache(cacheName);
+                IgniteCache cache1 = ignite(1).cache(cacheName);
+
+                final BlockingQueue<CacheEntryEvent<Integer, Integer>> evts0 = new LinkedBlockingQueue<>();
+                QueryCursor query = registerQuery(cache0, evts0);
+
+                final BlockingQueue<CacheEntryEvent<Integer, Integer>> evts1 = new LinkedBlockingQueue<>();
+                QueryCursor query1 = registerQuery(cache1, evts1);
+
+                final BlockingQueue<CacheEntryEvent<Integer, Integer>> evts2 = new LinkedBlockingQueue<>();
+                QueryCursor query2 = registerQuery(cache2, evts2);
+
+                Integer key = keyForNode(affinity(cache1), new AtomicInteger(0),
+                    grid(0).cluster().localNode());
+
+                // This update should CacheVersion with orderId == 3.
+                cache2.put(key, 42);
+
+                // The update will be have orderId == 2 and will be discarded.
+                cache1.put(key, 41);
+
+                assertEquals("Entry was updated.", 42, cache1.get(key));
+
+                cache0.remove(key);
+
+                assertNull("Entry was not removed.", cache1.get(key));
+
+                CacheEntryEvent<Integer, Integer> e = evts0.poll(200, TimeUnit.MILLISECONDS);
+
+                assertNotNull("Missed events.", e);
+                assertEquals("Invalid event.", EventType.CREATED, e.getEventType());
+                assertNull("Invalid event.", e.getOldValue());
+                assertEquals("Invalid event.", Integer.valueOf(42), e.getValue());
+
+                e = evts0.poll(200, TimeUnit.MILLISECONDS);
+
+                assertNotNull("Missed events.", e);
+                assertEquals("Invalid event.", EventType.REMOVED, e.getEventType());
+                assertEquals("Invalid event.", Integer.valueOf(42), e.getOldValue());
+
+                assertTrue("Received extra events.", evts1.isEmpty());
+                assertTrue("Received extra events.", evts2.isEmpty());
+
+                query.close();
+                query1.close();
+                query2.close();
+            }
+            finally {
+                grid(0).destroyCache(cacheName);
+            }
+        }
+    }
+
+    private QueryCursor registerQuery(IgniteCache cache, final BlockingQueue<CacheEntryEvent<Integer, Integer>> evts) {
+        ContinuousQuery qry = new ContinuousQuery();
+
+        qry.setLocalListener(new CacheEntryUpdatedListener() {
+            @Override public void onUpdated(Iterable iterable) throws CacheEntryListenerException {
+                for (Object o : iterable)
+                    evts.add((CacheEntryEvent)o);
+            }
+        });
+
+        qry.setLocal(true);
+
+        return cache.query(qry);
+    }
+
+    /**
+     * @param name Cache name.
+     * @return Cache configuration.
+     */
+    private CacheConfiguration cacheConfiguration(String name) {
+        return new CacheConfiguration(name)
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setAtomicWriteOrderMode(CacheAtomicWriteOrderMode.CLOCK)
+            .setBackups(0);
+    }
+
+    /**
+     *
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC)
+            throws IgniteSpiException {
+            Object msg0 = ((GridIoMessage)msg).message();
+
+            if (msg0 instanceof GridNearAtomicFullUpdateRequest) {
+                GridCacheVersion ver = ((GridNearAtomicFullUpdateRequest)msg0).updateVersion();
+
+                ver.globalTime(142);
+            }
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
index d2cb710edbeb5..2df38d767299b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
@@ -258,15 +258,15 @@ public void testFirstFilteredEvent() throws Exception {
                 qryClnCache.put(key, -1);
 
             qryClnCache.put(keys.get(0), 100);
-        }
 
-        GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override public boolean apply() {
-                return lsnr.evts.size() == 1;
-            }
-        }, 5000);
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return lsnr.evts.size() == 1;
+                }
+            }, 5000);
 
-        assertEquals(lsnr.evts.size(), 1);
+            assertEquals(1, lsnr.evts.size());
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
index 7d975f2a28209..2460a8045cd1f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
@@ -145,7 +145,37 @@ public void testAtomicOnheapTwoBackup() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicOnheapTwoBackupLocalQuery() throws Exception {
+        CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
+            ONHEAP_TIERED, PRIMARY_SYNC);
+
+        doOrderingTest(ccfg, false, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicOnheapTwoBackupLocalQueryAsync() throws Exception {
+        CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
+            ONHEAP_TIERED, PRIMARY_SYNC);
+
+        doOrderingTest(ccfg, true, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicOnheapTwoBackupLocalQueryAsyncFullSync() throws Exception {
+        CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
+            ONHEAP_TIERED, FULL_SYNC);
+
+        doOrderingTest(ccfg, false, true);
     }
 
     /**
@@ -155,7 +185,7 @@ public void testAtomicOffheapTwoBackup() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     /**
@@ -165,7 +195,7 @@ public void testAtomicOffheapValuesTwoBackup() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_VALUES, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     /**
@@ -175,7 +205,7 @@ public void testAtomicReplicatedOffheap() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(REPLICATED, 0, ATOMIC,
             OFFHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     /**
@@ -185,7 +215,7 @@ public void testTxOnheapTwoBackup() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, TRANSACTIONAL,
             ONHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     /**
@@ -195,7 +225,7 @@ public void testTxOnheapWithoutBackup() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 0, TRANSACTIONAL,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     /**
@@ -205,7 +235,7 @@ public void testTxOnheapWithoutBackupFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 0, TRANSACTIONAL,
             ONHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, false);
+        doOrderingTest(ccfg, false, false);
     }
 
     // ASYNC
@@ -217,7 +247,7 @@ public void testAtomicOnheapTwoBackupAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -227,7 +257,7 @@ public void testAtomicOnheapTwoBackupAsyncFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             ONHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -237,7 +267,7 @@ public void testAtomicOffheapTwoBackupAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -247,7 +277,7 @@ public void testAtomicOffheapTwoBackupAsyncFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -257,7 +287,7 @@ public void testAtomicOffheapValuesTwoBackupAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_VALUES, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -267,7 +297,7 @@ public void testAtomicOffheapValuesTwoBackupAsyncFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, ATOMIC,
             OFFHEAP_VALUES, FULL_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -277,7 +307,7 @@ public void testAtomicReplicatedAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(REPLICATED, 0, ATOMIC,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -287,7 +317,7 @@ public void testAtomicReplicatedAsyncFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(REPLICATED, 0, ATOMIC,
             ONHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -297,7 +327,7 @@ public void testAtomicReplicatedOffheapAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(REPLICATED, 0, ATOMIC,
             OFFHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -307,7 +337,7 @@ public void testAtomicOnheapWithoutBackupAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 0, ATOMIC,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -317,7 +347,7 @@ public void testTxOnheapTwoBackupAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 2, TRANSACTIONAL,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -327,7 +357,7 @@ public void testTxOnheapAsync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 0, TRANSACTIONAL,
             ONHEAP_TIERED, PRIMARY_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
@@ -337,17 +367,17 @@ public void testTxOnheapAsyncFullSync() throws Exception {
         CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 0, TRANSACTIONAL,
             ONHEAP_TIERED, FULL_SYNC);
 
-        doOrderingTest(ccfg, true);
+        doOrderingTest(ccfg, true, false);
     }
 
     /**
      * @param ccfg Cache configuration.
      * @param async Async filter.
+     * @param localQry Start local query.
+     *
      * @throws Exception If failed.
      */
-    protected void doOrderingTest(
-        final CacheConfiguration ccfg,
-        final boolean async)
+    protected void doOrderingTest(final CacheConfiguration ccfg, final boolean async, final boolean localQry)
         throws Exception {
         ignite(0).createCache(ccfg);
 
@@ -372,15 +402,17 @@ protected void doOrderingTest(
                         qry.setLocalListener(new TestCacheAsyncEventListener(queue, qryCntr));
 
                         qry.setRemoteFilterFactory(FactoryBuilder.factoryOf(
-                            new CacheTestRemoteFilterAsync(ccfg.getName())));
+                            new CacheTestRemoteFilterAsync(ccfg.getName(), localQry)));
                     }
                     else {
                         qry.setLocalListener(new TestCacheEventListener(queue, qryCntr));
 
                         qry.setRemoteFilterFactory(FactoryBuilder.factoryOf(
-                            new CacheTestRemoteFilter(ccfg.getName())));
+                            new CacheTestRemoteFilter(ccfg.getName(), localQry)));
                     }
 
+                    qry.setLocal(localQry);
+
                     rcvdEvts.add(queue);
 
                     IgniteCache<Object, Object> cache = grid(idx).cache(ccfg.getName());
@@ -448,14 +480,17 @@ protected void doOrderingTest(
 
             f.get(15, TimeUnit.SECONDS);
 
-            GridTestUtils.waitForCondition(new PA() {
+            boolean res = GridTestUtils.waitForCondition(new PA() {
                 @Override public boolean apply() {
-                    return qryCntr.get() >= ITERATION_CNT * threadCnt * LISTENER_CNT * NODES;
+                    return qryCntr.get() >= ITERATION_CNT * threadCnt * LISTENER_CNT *
+                        (localQry ? 1 : NODES);
                 }
             }, 1000L);
 
+            assertTrue("Failed to wait all events.", res);
+
             for (BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>> queue : rcvdEvts)
-                checkEvents(queue, ITERATION_CNT * threadCnt);
+                checkEvents(queue, ITERATION_CNT * threadCnt, localQry);
 
             assertFalse("Ordering invocations of filter broken.", fail);
         }
@@ -469,9 +504,12 @@ protected void doOrderingTest(
 
     /**
      * @param queue Event queue.
+     * @param localQry Local query flag.
      * @throws Exception If failed.
      */
-    private void checkEvents(BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>> queue, int expCnt)
+    private void checkEvents(BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>> queue,
+        int expCnt,
+        boolean localQry)
         throws Exception {
         CacheEntryEvent<QueryTestKey, QueryTestValue> evt;
         int cnt = 0;
@@ -483,11 +521,17 @@ private void checkEvents(BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestVa
 
             Integer preVal = vals.get(evt.getKey());
 
-            if (preVal == null)
-                assertEquals(new QueryTestValue(0), evt.getValue());
+            if (!localQry) {
+                if (preVal == null)
+                    assertEquals(new QueryTestValue(0), evt.getValue());
+                else {
+                    if (!new QueryTestValue(preVal + 1).equals(evt.getValue()))
+                        assertEquals("Key event: " + evt.getKey(), new QueryTestValue(preVal + 1), evt.getValue());
+                }
+            }
             else {
-                if (!new QueryTestValue(preVal + 1).equals(evt.getValue()))
-                    assertEquals("Key event: " + evt.getKey(), new QueryTestValue(preVal + 1), evt.getValue());
+                if (preVal != null)
+                    assertTrue("Evt out of order.", preVal < evt.getValue().val1);
             }
 
             vals.put(evt.getKey(), evt.getValue().val1);
@@ -495,7 +539,8 @@ private void checkEvents(BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestVa
             ++cnt;
         }
 
-        assertEquals(expCnt, cnt);
+        if (!localQry)
+            assertEquals(expCnt, cnt);
     }
 
     /** {@inheritDoc} */
@@ -510,9 +555,10 @@ private void checkEvents(BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestVa
     private static class CacheTestRemoteFilterAsync extends CacheTestRemoteFilter {
         /**
          * @param cacheName Cache name.
+         * @param localQry Local query.
          */
-        public CacheTestRemoteFilterAsync(String cacheName) {
-            super(cacheName);
+        public CacheTestRemoteFilterAsync(String cacheName, boolean localQry) {
+            super(cacheName, localQry);
         }
     }
 
@@ -531,11 +577,16 @@ private static class CacheTestRemoteFilter implements
         /** */
         private String cacheName;
 
+        /** */
+        private boolean localQry;
+
         /**
          * @param cacheName Cache name.
+         * @param localQry Local query flag.
          */
-        public CacheTestRemoteFilter(String cacheName) {
+        public CacheTestRemoteFilter(String cacheName, boolean localQry) {
             this.cacheName = cacheName;
+            this.localQry = localQry;
         }
 
         /** {@inheritDoc} */
@@ -544,7 +595,11 @@ public CacheTestRemoteFilter(String cacheName) {
                 QueryTestValue prevVal = prevVals.put(e.getKey(), e.getValue());
 
                 if (prevVal != null) {
-                    if (!new QueryTestValue(prevVal.val1 + 1).equals(e.getValue()))
+                    if (localQry) {
+                        if (prevVal.val1 > e.getValue().val1)
+                            fail = true;
+                    }
+                    else if (!new QueryTestValue(prevVal.val1 + 1).equals(e.getValue()))
                         fail = true;
                 }
             }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
index 6b2fea0a5ab55..3767bafe97199 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchAckTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchForceServerModeAckTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFilterListenerTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryClockModeConflictSelfTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryExecuteInPrimaryTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryAsyncFilterRandomOperationTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryFilterRandomOperationTest;
@@ -125,6 +126,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheContinuousQueryBackupQueueTest.class);
         suite.addTestSuite(IgniteCacheContinuousQueryNoUnsubscribeTest.class);
         suite.addTestSuite(ContinuousQueryPeerClassLoadingTest.class);
+        suite.addTestSuite(CacheContinuousQueryClockModeConflictSelfTest.class);
 
         return suite;
     }

From 5858efd406bb54352de14a0a7e7f21c2ac7bf899 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 16 Dec 2016 19:23:29 +0300
Subject: [PATCH 221/516] IGNITE-2412 - Do not acquire asyncSemaphore for
 synchronous operations (cherry-picked from master)

(cherry picked from commit 82b4073)
---
 .../processors/cache/GridCacheAdapter.java    | 703 +++++++-----------
 .../dht/atomic/GridDhtAtomicCache.java        | 469 +++++++-----
 .../dht/colocated/GridDhtColocatedCache.java  |  13 -
 .../distributed/near/GridNearAtomicCache.java |  65 +-
 .../local/atomic/GridLocalAtomicCache.java    | 173 +----
 ...bledMultiNodeLongTxTimeoutFullApiTest.java |   2 +-
 ...ckMessageSystemPoolStarvationSelfTest.java |  13 +-
 7 files changed, 596 insertions(+), 842 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index ac1d2682d5f2e..47c9998239777 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -248,16 +248,16 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
 
     /** Grid configuration. */
     @GridToStringExclude
-    protected IgniteConfiguration gridCfg;
+    private IgniteConfiguration gridCfg;
 
     /** Cache metrics. */
     protected CacheMetricsImpl metrics;
 
     /** Cache localMxBean. */
-    protected CacheMetricsMXBean localMxBean;
+    private CacheMetricsMXBean locMxBean;
 
     /** Cache mxBean. */
-    protected CacheMetricsMXBean clusterMxBean;
+    private CacheMetricsMXBean clusterMxBean;
 
     /** Logger. */
     protected IgniteLogger log;
@@ -331,7 +331,7 @@ protected GridCacheAdapter(final GridCacheContext<K, V> ctx, @Nullable GridCache
 
         metrics = new CacheMetricsImpl(ctx);
 
-        localMxBean = new CacheLocalMetricsMXBeanImpl(this);
+        locMxBean = new CacheLocalMetricsMXBeanImpl(this);
         clusterMxBean = new CacheClusterMetricsMXBeanImpl(this);
 
         FileSystemConfiguration[] igfsCfgs = gridCfg.getFileSystemConfiguration();
@@ -456,49 +456,49 @@ public boolean isDht() {
     public abstract GridCachePreloader preloader();
 
     /** {@inheritDoc} */
-    @Override public Affinity<K> affinity() {
+    @Override public final Affinity<K> affinity() {
         return aff;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings({"unchecked", "RedundantCast"})
-    @Override public <K1, V1> IgniteInternalCache<K1, V1> cache() {
+    @Override public final <K1, V1> IgniteInternalCache<K1, V1> cache() {
         return (IgniteInternalCache<K1, V1>)this;
     }
 
     /** {@inheritDoc} */
-    @Override public GridCacheProxyImpl<K, V> forSubjectId(UUID subjId) {
+    @Override public final GridCacheProxyImpl<K, V> forSubjectId(UUID subjId) {
         CacheOperationContext opCtx = new CacheOperationContext(false, subjId, false, null, false, null);
 
         return new GridCacheProxyImpl<>(ctx, this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Override public boolean skipStore() {
+    @Override public final boolean skipStore() {
         return false;
     }
 
     /** {@inheritDoc} */
-    @Override public GridCacheProxyImpl<K, V> setSkipStore(boolean skipStore) {
+    @Override public final GridCacheProxyImpl<K, V> setSkipStore(boolean skipStore) {
         CacheOperationContext opCtx = new CacheOperationContext(true, null, false, null, false, null);
 
         return new GridCacheProxyImpl<>(ctx, this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Override public <K1, V1> GridCacheProxyImpl<K1, V1> keepBinary() {
+    @Override public final <K1, V1> GridCacheProxyImpl<K1, V1> keepBinary() {
         CacheOperationContext opCtx = new CacheOperationContext(false, null, true, null, false, null);
 
         return new GridCacheProxyImpl<>((GridCacheContext<K1, V1>)ctx, (GridCacheAdapter<K1, V1>)this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public ExpiryPolicy expiry() {
+    @Nullable @Override public final ExpiryPolicy expiry() {
         return null;
     }
 
     /** {@inheritDoc} */
-    @Override public GridCacheProxyImpl<K, V> withExpiryPolicy(ExpiryPolicy plc) {
+    @Override public final GridCacheProxyImpl<K, V> withExpiryPolicy(ExpiryPolicy plc) {
         assert !CU.isUtilityCache(ctx.name());
         assert !CU.isAtomicsCache(ctx.name());
         assert !CU.isMarshallerCache(ctx.name());
@@ -509,14 +509,14 @@ public boolean isDht() {
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalCache<K, V> withNoRetries() {
+    @Override public final IgniteInternalCache<K, V> withNoRetries() {
         CacheOperationContext opCtx = new CacheOperationContext(false, null, false, null, true, null);
 
         return new GridCacheProxyImpl<>(ctx, this, opCtx);
     }
 
     /** {@inheritDoc} */
-    @Override public CacheConfiguration configuration() {
+    @Override public final CacheConfiguration configuration() {
         return ctx.config();
     }
 
@@ -617,7 +617,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean isEmpty() {
+    @Override public final boolean isEmpty() {
         try {
             return localSize(CachePeekModes.ONHEAP_ONLY) == 0;
         }
@@ -627,7 +627,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean containsKey(K key) {
+    @Override public final boolean containsKey(K key) {
         try {
             return containsKeyAsync(key).get();
         }
@@ -654,7 +654,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean containsKeys(Collection<? extends K> keys) {
+    @Override public final boolean containsKeys(Collection<? extends K> keys) {
         try {
             return containsKeysAsync(keys).get();
         }
@@ -664,7 +664,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> containsKeysAsync(final Collection<? extends K> keys) {
+    @Override public final IgniteInternalFuture<Boolean> containsKeysAsync(final Collection<? extends K> keys) {
         A.notNull(keys, "keys");
 
         return getAllAsync(
@@ -695,7 +695,7 @@ public void onKernalStop() {
     }
 
     /** {@inheritDoc} */
-    @Override public Iterable<Cache.Entry<K, V>> localEntries(CachePeekMode[] peekModes) throws IgniteCheckedException {
+    @Override public final Iterable<Cache.Entry<K, V>> localEntries(CachePeekMode[] peekModes) throws IgniteCheckedException {
         assert peekModes != null;
 
         ctx.checkSecurity(SecurityPermission.CACHE_READ);
@@ -752,7 +752,7 @@ public String toString() {
 
     /** {@inheritDoc} */
     @SuppressWarnings("ForLoopReplaceableByForEach")
-    @Nullable @Override public V localPeek(K key,
+    @Nullable @Override public final V localPeek(K key,
         CachePeekMode[] peekModes,
         @Nullable IgniteCacheExpiryPolicy plc)
         throws IgniteCheckedException {
@@ -894,7 +894,7 @@ public String toString() {
      *
      * @param ldr Class loader to undeploy.
      */
-    public void onUndeploy(ClassLoader ldr) {
+    public final void onUndeploy(ClassLoader ldr) {
         ctx.deploy().onUndeploy(ldr, context());
     }
 
@@ -903,7 +903,7 @@ public void onUndeploy(ClassLoader ldr) {
      * @param key Entry key.
      * @return Entry or <tt>null</tt>.
      */
-    @Nullable public GridCacheEntryEx peekEx(KeyCacheObject key) {
+    @Nullable public final GridCacheEntryEx peekEx(KeyCacheObject key) {
         return entry0(key, ctx.affinity().affinityTopologyVersion(), false, false);
     }
 
@@ -912,7 +912,7 @@ public void onUndeploy(ClassLoader ldr) {
      * @param key Entry key.
      * @return Entry or <tt>null</tt>.
      */
-    @Nullable public GridCacheEntryEx peekEx(Object key) {
+    @Nullable public final GridCacheEntryEx peekEx(Object key) {
         return entry0(ctx.toCacheKeyObject(key), ctx.affinity().affinityTopologyVersion(), false, false);
     }
 
@@ -920,7 +920,7 @@ public void onUndeploy(ClassLoader ldr) {
      * @param key Entry key.
      * @return Entry (never {@code null}).
      */
-    public GridCacheEntryEx entryEx(Object key) {
+    public final GridCacheEntryEx entryEx(Object key) {
         return entryEx(ctx.toCacheKeyObject(key), false);
     }
 
@@ -928,7 +928,7 @@ public GridCacheEntryEx entryEx(Object key) {
      * @param key Entry key.
      * @return Entry (never {@code null}).
      */
-    public GridCacheEntryEx entryEx(KeyCacheObject key) {
+    public final GridCacheEntryEx entryEx(KeyCacheObject key) {
         return entryEx(key, false);
     }
 
@@ -983,24 +983,24 @@ public GridCacheEntryEx entryEx(KeyCacheObject key, AffinityTopologyVersion topV
     /**
      * @return Set of internal cached entry representations.
      */
-    public Iterable<? extends GridCacheEntryEx> entries() {
+    public final Iterable<? extends GridCacheEntryEx> entries() {
         return allEntries();
     }
 
     /**
      * @return Set of internal cached entry representations.
      */
-    public Iterable<? extends GridCacheEntryEx> allEntries() {
+    public final Iterable<? extends GridCacheEntryEx> allEntries() {
         return map.entries();
     }
 
     /** {@inheritDoc} */
-    @Override public Set<Cache.Entry<K, V>> entrySet() {
+    @Override public final Set<Cache.Entry<K, V>> entrySet() {
         return entrySet((CacheEntryPredicate[])null);
     }
 
     /** {@inheritDoc} */
-    @Override public Set<Cache.Entry<K, V>> entrySetx(final CacheEntryPredicate... filter) {
+    @Override public final Set<Cache.Entry<K, V>> entrySetx(final CacheEntryPredicate... filter) {
         boolean keepBinary = ctx.keepBinary();
 
         return new EntrySet(map.entrySet(filter), keepBinary);
@@ -1012,17 +1012,17 @@ public Iterable<? extends GridCacheEntryEx> allEntries() {
     }
 
     /** {@inheritDoc} */
-    @Override public Set<K> keySet() {
+    @Override public final Set<K> keySet() {
         return new KeySet(map.entrySet());
     }
 
     /** {@inheritDoc} */
-    @Override public Set<K> keySetx() {
+    @Override public final Set<K> keySetx() {
         return keySet();
     }
 
     /** {@inheritDoc} */
-    @Override public Set<K> primaryKeySet() {
+    @Override public final Set<K> primaryKeySet() {
         return new KeySet(map.entrySet(CU.cachePrimary(ctx.grid().affinity(ctx.name()), ctx.localNode())));
     }
 
@@ -1044,7 +1044,7 @@ public Iterable<? extends GridCacheEntryEx> allEntries() {
      * @param filter Filters.
      * @return Collection of cached values.
      */
-    public Iterable<V> values(final CacheEntryPredicate... filter) {
+    public final Iterable<V> values(final CacheEntryPredicate... filter) {
         return new Iterable<V>() {
             @Override public Iterator<V> iterator() {
                 return new Iterator<V>() {
@@ -1070,12 +1070,12 @@ public Iterable<V> values(final CacheEntryPredicate... filter) {
      *
      * @param key Entry key.
      */
-    public void removeIfObsolete(KeyCacheObject key) {
+    public final void removeIfObsolete(KeyCacheObject key) {
         assert key != null;
 
         GridCacheMapEntry entry = map.getEntry(key);
 
-        if (entry.obsolete())
+        if (entry != null && entry.obsolete())
             removeEntry(entry);
     }
 
@@ -1259,11 +1259,11 @@ public void clearLocally(Collection<KeyCacheObject> keys, boolean readers) {
     /**
      * @param entry Removes entry from cache if currently mapped value is the same as passed.
      */
-    public void removeEntry(GridCacheEntryEx entry) {
-        boolean removed = map.removeEntry(entry);
+    public final void removeEntry(GridCacheEntryEx entry) {
+        boolean rmvd = map.removeEntry(entry);
 
         if (log.isDebugEnabled()) {
-            if (removed)
+            if (rmvd)
                 log.debug("Removed entry from cache: " + entry);
             else
                 log.debug("Remove will not be done for key (entry got replaced or removed): " + entry.key());
@@ -1298,7 +1298,7 @@ private boolean evictx(K key, GridCacheVersion ver,
     }
 
     /** {@inheritDoc} */
-    @Override public V getForcePrimary(K key) throws IgniteCheckedException {
+    @Override public final V getForcePrimary(K key) throws IgniteCheckedException {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(
@@ -1315,7 +1315,7 @@ private boolean evictx(K key, GridCacheVersion ver,
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getForcePrimaryAsync(final K key) {
+    @Override public final IgniteInternalFuture<V> getForcePrimaryAsync(final K key) {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(
@@ -1336,7 +1336,7 @@ private boolean evictx(K key, GridCacheVersion ver,
     }
 
     /** {@inheritDoc} */
-    public V getTopologySafe(K key) throws IgniteCheckedException {
+    public final V getTopologySafe(K key) throws IgniteCheckedException {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(
@@ -1353,12 +1353,12 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public Map<K, V> getAllOutTx(Set<? extends K> keys) throws IgniteCheckedException {
+    @Nullable @Override public final Map<K, V> getAllOutTx(Set<? extends K> keys) throws IgniteCheckedException {
         return getAllOutTxAsync(keys).get();
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Map<K, V>> getAllOutTxAsync(Set<? extends K> keys) {
+    @Override public final IgniteInternalFuture<Map<K, V>> getAllOutTxAsync(Set<? extends K> keys) {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
         return getAllAsync(keys,
@@ -1372,15 +1372,6 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
             false);
     }
 
-    /**
-     * @param key Key.
-     * @param topVer Topology version.
-     * @return Entry.
-     */
-    @Nullable protected GridCacheEntryEx entryExSafe(KeyCacheObject key, AffinityTopologyVersion topVer) {
-        return entryEx(key);
-    }
-
     /** {@inheritDoc} */
     @Nullable @Override public V get(K key) throws IgniteCheckedException {
         A.notNull(key, "key");
@@ -1521,14 +1512,14 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
     }
 
     /** {@inheritDoc} */
-    @Override public Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
+    @Override public final Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
         boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         long start = statsEnabled ? System.nanoTime() : 0L;
 
-        Map<K, V> map = getAll(keys, !ctx.keepBinary(), false);
+        Map<K, V> map = getAll0(keys, !ctx.keepBinary(), false);
 
         if (ctx.config().getInterceptor() != null)
             map = interceptGet(keys, map);
@@ -1548,7 +1539,7 @@ public V getTopologySafe(K key) throws IgniteCheckedException {
 
         long start = statsEnabled ? System.nanoTime() : 0L;
 
-        Map<K, EntryGetResult> map = (Map<K, EntryGetResult>)getAll(keys, !ctx.keepBinary(), true);
+        Map<K, EntryGetResult> map = (Map<K, EntryGetResult>)getAll0(keys, !ctx.keepBinary(), true);
 
         Collection<CacheEntry<K, V>> res = new HashSet<>();
 
@@ -2156,7 +2147,7 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndPut(K key, V val) throws IgniteCheckedException {
+    @Override public final V getAndPut(K key, V val) throws IgniteCheckedException {
         return getAndPut(key, val, null);
     }
 
@@ -2178,7 +2169,24 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
         if (keyCheck)
             validateCacheKey(key);
 
-        V prevVal = syncOp(new SyncOp<V>(true) {
+        V prevVal = getAndPut0(key, val, filter);
+
+        if (statsEnabled)
+            metrics0().addPutAndGetTimeNanos(System.nanoTime() - start);
+
+        return prevVal;
+    }
+
+    /**
+     * @param key Key.
+     * @param val Value.
+     * @param filter Optional filter.
+     * @return Previous value.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected V getAndPut0(final K key, final V val, @Nullable final CacheEntryPredicate filter)
+        throws IgniteCheckedException {
+        return syncOp(new SyncOp<V>(true) {
             @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return (V)tx.putAsync(ctx, null, key, val, true, filter).get().value();
             }
@@ -2187,15 +2195,10 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                 return "put [key=" + key + ", val=" + val + ", filter=" + filter + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addPutAndGetTimeNanos(System.nanoTime() - start);
-
-        return prevVal;
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutAsync(K key, V val) {
+    @Override public final IgniteInternalFuture<V> getAndPutAsync(K key, V val) {
         return getAndPutAsync(key, val, null);
     }
 
@@ -2205,11 +2208,16 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
      * @param filter Filter.
      * @return Put operation future.
      */
-    public IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+    protected final IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
         final boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
 
+        A.notNull(key, "key", val, "val");
+
+        if (keyCheck)
+            validateCacheKey(key);
+
         IgniteInternalFuture<V> fut = getAndPutAsync0(key, val, filter);
 
         if (statsEnabled)
@@ -2224,13 +2232,10 @@ public IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntry
      * @param filter Optional filter.
      * @return Put operation future.
      */
-    public IgniteInternalFuture<V> getAndPutAsync0(final K key, final V val,
-        @Nullable final CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
+    public IgniteInternalFuture<V> getAndPutAsync0(final K key,
+        final V val,
+        @Nullable final CacheEntryPredicate filter)
+    {
         return asyncOp(new AsyncOp<V>() {
             @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAsync(ctx, readyTopVer, key, val, true, filter)
@@ -2244,7 +2249,7 @@ public IgniteInternalFuture<V> getAndPutAsync0(final K key, final V val,
     }
 
     /** {@inheritDoc} */
-    @Override public boolean put(final K key, final V val) throws IgniteCheckedException {
+    @Override public final boolean put(final K key, final V val) throws IgniteCheckedException {
         return put(key, val, null);
     }
 
@@ -2268,7 +2273,26 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
         if (keyCheck)
             validateCacheKey(key);
 
-        Boolean stored = syncOp(new SyncOp<Boolean>(true) {
+        boolean stored = put0(key, val, filter);
+
+        if (statsEnabled && stored)
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
+
+        return stored;
+    }
+
+    /**
+     * @param key Key.
+     * @param val Value.
+     * @param filter Filter.
+     * @return {@code True} if optional filter passed and value was stored in cache,
+     *      {@code false} otherwise. Note that this method will return {@code true} if filter is not
+     *      specified.
+     * @throws IgniteCheckedException If put operation failed.
+     */
+    protected boolean put0(final K key, final V val, final CacheEntryPredicate filter)
+        throws IgniteCheckedException {
+        Boolean res = syncOp(new SyncOp<Boolean>(true) {
             @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return tx.putAsync(ctx, null, key, val, false, filter).get().success();
             }
@@ -2278,10 +2302,9 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
             }
         });
 
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
+        assert res != null;
 
-        return stored;
+        return res;
     }
 
     /** {@inheritDoc} */
@@ -2323,7 +2346,7 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public <T> EntryProcessorResult<T> invoke(@Nullable AffinityTopologyVersion topVer,
+    @Nullable @Override public final <T> EntryProcessorResult<T> invoke(@Nullable AffinityTopologyVersion topVer,
         K key,
         EntryProcessor<K, V, T> entryProcessor,
         Object... args) throws IgniteCheckedException {
@@ -2556,7 +2579,7 @@ private <T> EntryProcessorResult<T> invoke0(
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putAsync(K key, V val) {
+    @Override public final IgniteInternalFuture<Boolean> putAsync(K key, V val) {
         return putAsync(key, val, null);
     }
 
@@ -2566,7 +2589,12 @@ private <T> EntryProcessorResult<T> invoke0(
      * @param filter Filter.
      * @return Put future.
      */
-    public IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+    public final IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+        A.notNull(key, "key", val, "val");
+
+        if (keyCheck)
+            validateCacheKey(key);
+
         final boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
@@ -2587,11 +2615,6 @@ public IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntry
      */
     public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         @Nullable final CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
         return asyncOp(new AsyncOp<Boolean>() {
             @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAsync(ctx,
@@ -2619,267 +2642,82 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public V getAndPutIfAbsent(final K key, final V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return syncOp(new SyncOp<V>(true) {
-            @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return (V)tx.putAsync(ctx, null, key, val, true, ctx.noVal()).get().value();
-            }
-
-            @Override public String toString() {
-                return "putIfAbsent [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Nullable @Override public final V getAndPutIfAbsent(final K key, final V val) throws IgniteCheckedException {
+        return getAndPut(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
-            @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx, readyTopVer, key, val, true, ctx.noVal())
-                    .chain((IgniteClosure<IgniteInternalFuture<GridCacheReturn>, V>)RET2VAL);
-            }
-
-            @Override public String toString() {
-                return "putIfAbsentAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
+    @Override public final IgniteInternalFuture<V> getAndPutIfAbsentAsync(final K key, final V val) {
+        return getAndPutAsync(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(final K key, final V val) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        Boolean stored = syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return tx.putAsync(ctx, null, key, val, false, ctx.noVal()).get().success();
-            }
-
-            @Override public String toString() {
-                return "putxIfAbsent [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled && stored)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
-
-        return stored;
+    @Override public final boolean putIfAbsent(final K key, final V val) throws IgniteCheckedException {
+        return put(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx,
-                    readyTopVer,
-                    key,
-                    val,
-                    false,
-                    ctx.noVal()).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "putxIfAbsentAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+    @Override public final IgniteInternalFuture<Boolean> putIfAbsentAsync(final K key, final V val) {
+        return putAsync(key, val, ctx.noVal());
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public V getAndReplace(final K key, final V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return syncOp(new SyncOp<V>(true) {
-            @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return (V)tx.putAsync(ctx, null, key, val, true, ctx.hasVal()).get().value();
-            }
-
-            @Override public String toString() {
-                return "replace [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Nullable @Override public final V getAndReplace(final K key, final V val) throws IgniteCheckedException {
+        return getAndPut(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
-            @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx, readyTopVer, key, val, true, ctx.hasVal()).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, V>)RET2VAL);
-            }
-
-            @Override public String toString() {
-                return "replaceAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutAndGetTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
+    @Override public final IgniteInternalFuture<V> getAndReplaceAsync(final K key, final V val) {
+        return getAndPutAsync(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public boolean replace(final K key, final V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                return tx.putAsync(ctx, null, key, val, false, ctx.hasVal()).get().success();
-            }
-
-            @Override public String toString() {
-                return "replacex [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Override public final boolean replace(final K key, final V val) throws IgniteCheckedException {
+        return put(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(final K key, final V val) {
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        return asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                return tx.putAsync(ctx, readyTopVer, key, val, false, ctx.hasVal()).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>) RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "replacexAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
+    @Override public final IgniteInternalFuture<Boolean> replaceAsync(final K key, final V val) {
+        return putAsync(key, val, ctx.hasVal());
     }
 
     /** {@inheritDoc} */
-    @Override public boolean replace(final K key, final V oldVal, final V newVal) throws IgniteCheckedException {
-        A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
-        if (keyCheck)
-            validateCacheKey(key);
+    @Override public final boolean replace(final K key, final V oldVal, final V newVal) throws IgniteCheckedException {
+        A.notNull(oldVal, "oldVal");
 
-        return syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled())
-                    ctx.deploy().registerClass(oldVal);
-
-                return tx.putAsync(ctx, null, key, newVal, false, ctx.equalsVal(oldVal)).get()
-                    .success();
-            }
-
-            @Override public String toString() {
-                return "replace [key=" + key + ", oldVal=" + oldVal + ", newVal=" + newVal + ']';
-            }
-        });
+        return put(key, newVal, ctx.equalsVal(oldVal));
     }
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<Boolean> replaceAsync(final K key, final V oldVal, final V newVal) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
-        if (keyCheck)
-            validateCacheKey(key);
-
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled()) {
-                    try {
-                        ctx.deploy().registerClass(oldVal);
-                    }
-                    catch (IgniteCheckedException e) {
-                        return new GridFinishedFuture<>(e);
-                    }
-                }
+        A.notNull(oldVal, "oldVal");
 
-                return tx.putAsync(ctx, readyTopVer, key, newVal, false, ctx.equalsVal(oldVal)).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "replaceAsync [key=" + key + ", oldVal=" + oldVal + ", newVal=" + newVal + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutAndGetTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+        return putAsync(key, newVal, ctx.equalsVal(oldVal));
     }
 
     /** {@inheritDoc} */
     @Override public void putAll(@Nullable final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        if (F.isEmpty(m))
+            return;
+
         boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         long start = statsEnabled ? System.nanoTime() : 0L;
 
-        if (F.isEmpty(m))
-            return;
-
         if (keyCheck)
             validateCacheKeys(m.keySet());
 
+        putAll0(m);
+
+        if (statsEnabled)
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
+    }
+
+    /**
+     * @param m Map.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void putAll0(final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
         syncOp(new SyncInOp(m.size() == 1) {
             @Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 tx.putAllAsync(ctx, null, m, false).get();
@@ -2889,9 +2727,6 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "putAll [map=" + m + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
@@ -2902,6 +2737,14 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKeys(m.keySet());
 
+        return putAllAsync0(m);
+    }
+
+    /**
+     * @param m Map.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<?> putAllAsync0(final Map<? extends K, ? extends V> m) {
         return asyncOp(new AsyncOp(m.keySet()) {
             @Override public IgniteInternalFuture<?> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAllAsync(ctx,
@@ -2927,11 +2770,25 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKey(key);
 
+        V prevVal = getAndRemove0(key);
+
+        if (statsEnabled)
+            metrics0().addRemoveAndGetTimeNanos(System.nanoTime() - start);
+
+        return prevVal;
+    }
+
+    /**
+     * @param key Key.
+     * @return Previous value.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected V getAndRemove0(final K key) throws IgniteCheckedException {
         final boolean keepBinary = ctx.keepBinary();
 
-        V prevVal = syncOp(new SyncOp<V>(true) {
+        return syncOp(new SyncOp<V>(true) {
             @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                K key0 = keepBinary ? (K)ctx.toCacheKeyObject(key) : key;
+                K key0 = keepBinary ? (K) ctx.toCacheKeyObject(key) : key;
 
                 V ret = tx.removeAllAsync(ctx,
                     null,
@@ -2941,9 +2798,9 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                     /*singleRmv*/false).get().value();
 
                 if (ctx.config().getInterceptor() != null) {
-                    K key = keepBinary ? (K)ctx.unwrapBinaryIfNeeded(key0, true, false) : key0;
+                    K key = keepBinary ? (K) ctx.unwrapBinaryIfNeeded(key0, true, false) : key0;
 
-                    return (V)ctx.config().getInterceptor().onBeforeRemove(new CacheEntryImpl(key, ret)).get2();
+                    return (V) ctx.config().getInterceptor().onBeforeRemove(new CacheEntryImpl(key, ret)).get2();
                 }
 
                 return ret;
@@ -2953,11 +2810,6 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "remove [key=" + key + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addRemoveAndGetTimeNanos(System.nanoTime() - start);
-
-        return prevVal;
     }
 
     /** {@inheritDoc} */
@@ -2971,7 +2823,20 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
+        IgniteInternalFuture<V> fut = getAndRemoveAsync0(key);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
+    }
+
+    /**
+     * @param key Key.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
+        return asyncOp(new AsyncOp<V>() {
             @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 // TODO should we invoke interceptor here?
                 return tx.removeAllAsync(ctx,
@@ -2986,11 +2851,6 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "removeAsync [key=" + key + ']';
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
     }
 
     /** {@inheritDoc} */
@@ -3023,6 +2883,17 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKeys(keys);
 
+        removeAll0(keys);
+
+        if (statsEnabled)
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+    }
+
+    /**
+     * @param keys Keys.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheckedException {
         syncOp(new SyncInOp(keys.size() == 1) {
             @Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 tx.removeAllAsync(ctx,
@@ -3037,24 +2908,34 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "removeAll [keys=" + keys + ']';
             }
         });
-
-        if (statsEnabled)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<?> removeAllAsync(@Nullable final Collection<? extends K> keys) {
+        if (F.isEmpty(keys))
+            return new GridFinishedFuture<Object>();
+
         final boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         final long start = statsEnabled ? System.nanoTime() : 0L;
 
-        if (F.isEmpty(keys))
-            return new GridFinishedFuture<Object>();
-
         if (keyCheck)
             validateCacheKeys(keys);
 
-        IgniteInternalFuture<Object> fut = asyncOp(new AsyncOp(keys) {
+        IgniteInternalFuture<Object> fut = removeAllAsync0(keys);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+
+        return fut;
+    }
+
+    /**
+     * @param keys Keys.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<Object> removeAllAsync0(final Collection<? extends K> keys) {
+        return asyncOp(new AsyncOp(keys) {
             @Override public IgniteInternalFuture<?> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.removeAllAsync(ctx,
                     readyTopVer,
@@ -3068,15 +2949,20 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                 return "removeAllAsync [keys=" + keys + ']';
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
-
-        return fut;
     }
 
     /** {@inheritDoc} */
     @Override public boolean remove(final K key) throws IgniteCheckedException {
+        return remove(key, (CacheEntryPredicate)null);
+    }
+
+    /**
+     * @param key Key.
+     * @param filter Filter.
+     * @return {@code True} if entry was removed.
+     * @throws IgniteCheckedException If failed.
+     */
+    public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
         boolean statsEnabled = ctx.config().isStatisticsEnabled();
 
         long start = statsEnabled ? System.nanoTime() : 0L;
@@ -3086,13 +2972,27 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (keyCheck)
             validateCacheKey(key);
 
-        boolean rmv = syncOp(new SyncOp<Boolean>(true) {
+        boolean rmv = remove0(key, filter);
+
+        if (statsEnabled && rmv)
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+
+        return rmv;
+    }
+
+    /**
+     * @param key Key.
+     * @return {@code True} if entry was removed.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected boolean remove0(final K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
+        Boolean res = syncOp(new SyncOp<Boolean>(true) {
             @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return tx.removeAllAsync(ctx,
                     null,
                     Collections.singletonList(key),
                     /*retval*/false,
-                    null,
+                    filter,
                     /*singleRmv*/true).get().success();
             }
 
@@ -3101,10 +3001,9 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
             }
         });
 
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+        assert res != null;
 
-        return rmv;
+        return res;
     }
 
     /** {@inheritDoc} */
@@ -3129,7 +3028,21 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
+        IgniteInternalFuture<Boolean> fut = removeAsync0(key, filter);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
+
+        return fut;
+    }
+
+    /**
+     * @param key Key.
+     * @param filter Filter.
+     * @return Future.
+     */
+    protected IgniteInternalFuture<Boolean> removeAsync0(final K key, @Nullable final CacheEntryPredicate filter) {
+        return asyncOp(new AsyncOp<Boolean>() {
             @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.removeAllAsync(ctx,
                     readyTopVer,
@@ -3144,11 +3057,6 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
                 return "removeAsync [key=" + key + ", filter=" + filter + ']';
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
     }
 
     /** {@inheritDoc} */
@@ -3190,86 +3098,21 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
     }
 
     /** {@inheritDoc} */
-    @Override public boolean remove(final K key, final V val) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
+    @Override public final boolean remove(final K key, final V val) throws IgniteCheckedException {
+        A.notNull(val, "val");
 
-        if (keyCheck)
-            validateCacheKey(key);
-
-        boolean rmv = syncOp(new SyncOp<Boolean>(true) {
-            @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled())
-                    ctx.deploy().registerClass(val);
-
-                return tx.removeAllAsync(ctx,
-                    null,
-                    Collections.singletonList(key),
-                    /*retval*/false,
-                    ctx.equalsVal(val),
-                    /*singleRmv*/false).get().success();
-            }
-
-            @Override public String toString() {
-                return "remove [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
-
-        return rmv;
+        return remove(key, ctx.equalsVal(val));
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(final K key, final V val) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key", val, "val");
-
-        if (keyCheck)
-            validateCacheKey(key);
+    @Override public final IgniteInternalFuture<Boolean> removeAsync(final K key, final V val) {
+        A.notNull(key, "val");
 
-        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
-            @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
-                // Register before hiding in the filter.
-                if (ctx.deploymentEnabled()) {
-                    try {
-                        ctx.deploy().registerClass(val);
-                    }
-                    catch (IgniteCheckedException e) {
-                        return new GridFinishedFuture<>(e);
-                    }
-                }
-
-                return tx.removeAllAsync(ctx,
-                    readyTopVer,
-                    Collections.singletonList(key),
-                    /*retval*/false,
-                    ctx.equalsVal(val),
-                    /*singleRmv*/false).chain(
-                    (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
-            }
-
-            @Override public String toString() {
-                return "removeAsync [key=" + key + ", val=" + val + ']';
-            }
-        });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+        return removeAsync(key, ctx.equalsVal(val));
     }
 
     /** {@inheritDoc} */
-    @Override public CacheMetrics clusterMetrics() {
+    @Override public final CacheMetrics clusterMetrics() {
         return clusterMetrics(ctx.grid().cluster().forDataNodes(ctx.name()));
     }
 
@@ -3298,7 +3141,7 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
 
     /** {@inheritDoc} */
     @Override public CacheMetricsMXBean localMxBean() {
-        return localMxBean;
+        return locMxBean;
     }
 
     /** {@inheritDoc} */
@@ -4647,7 +4490,7 @@ protected void saveFuture(final FutureHolder holder, IgniteInternalFuture<?> fut
     /**
      * Releases asynchronous operations permit, if limited.
      */
-    protected void asyncOpRelease() {
+    private void asyncOpRelease() {
         if (asyncOpsSem != null)
             asyncOpsSem.release();
     }
@@ -4813,12 +4656,10 @@ public Set<Cache.Entry<K, V>> entrySet(@Nullable CacheEntryPredicate... filter)
      * @return Cached value.
      * @throws IgniteCheckedException If failed.
      */
-    @Nullable public V get(K key, boolean deserializeBinary, final boolean needVer) throws IgniteCheckedException {
-        checkJta();
-
+    @Nullable public final V get(K key, boolean deserializeBinary, final boolean needVer) throws IgniteCheckedException {
         String taskName = ctx.kernalContext().job().currentTaskName();
 
-        return get(key, taskName, deserializeBinary, needVer);
+        return get0(key, taskName, deserializeBinary, needVer);
     }
 
     /**
@@ -4829,11 +4670,13 @@ public Set<Cache.Entry<K, V>> entrySet(@Nullable CacheEntryPredicate... filter)
      * @return Cached value.
      * @throws IgniteCheckedException If failed.
      */
-    protected V get(
+    protected V get0(
         final K key,
         String taskName,
         boolean deserializeBinary,
         boolean needVer) throws IgniteCheckedException {
+        checkJta();
+
         try {
             return getAsync(key,
                 !ctx.config().isReadFromBackup(),
@@ -4887,7 +4730,7 @@ public final IgniteInternalFuture<V> getAsync(final K key, boolean deserializeBi
      * @return Map of cached values.
      * @throws IgniteCheckedException If read failed.
      */
-    public Map<K, V> getAll(Collection<? extends K> keys, boolean deserializeBinary,
+    protected Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary,
         boolean needVer) throws IgniteCheckedException {
         checkJta();
 
@@ -4942,7 +4785,7 @@ public void forceKeyCheck() {
      * @param key Cache key.
      * @throws IllegalArgumentException If validation fails.
      */
-    protected void validateCacheKey(Object key) {
+    protected final void validateCacheKey(Object key) {
         if (keyCheck) {
             CU.validateCacheKey(key);
 
@@ -4957,7 +4800,7 @@ protected void validateCacheKey(Object key) {
      * @param keys Cache keys.
      * @throws IgniteException If validation fails.
      */
-    protected void validateCacheKeys(Iterable<?> keys) {
+    protected final void validateCacheKeys(Iterable<?> keys) {
         if (keys == null)
             return;
 
@@ -4978,7 +4821,7 @@ protected void validateCacheKeys(Iterable<?> keys) {
      * @param deserializeBinary Deserialize binary flag.
      * @return Public API iterator.
      */
-    protected Iterator<Cache.Entry<K, V>> iterator(final Iterator<? extends GridCacheEntryEx> it,
+    protected final Iterator<Cache.Entry<K, V>> iterator(final Iterator<? extends GridCacheEntryEx> it,
         final boolean deserializeBinary) {
         return new Iterator<Cache.Entry<K, V>>() {
             {
@@ -5296,7 +5139,7 @@ private static PeekModes parsePeekModes(CachePeekMode[] peekModes, boolean prima
      * @param plc Explicitly specified expiry policy for cache operation.
      * @return Expiry policy wrapper.
      */
-    @Nullable public IgniteCacheExpiryPolicy expiryPolicy(@Nullable ExpiryPolicy plc) {
+    @Nullable public final IgniteCacheExpiryPolicy expiryPolicy(@Nullable ExpiryPolicy plc) {
         if (plc == null)
             plc = ctx.expiry();
 
@@ -5421,7 +5264,7 @@ public IgniteInternalFuture<T> op(final IgniteTxLocalAdapter tx, CacheOperationC
          * @param opCtx Operation context.
          * @return Operation future.
          */
-        protected IgniteInternalFuture<T> waitTopologyFuture(IgniteInternalFuture<?> topFut,
+        private IgniteInternalFuture<T> waitTopologyFuture(IgniteInternalFuture<?> topFut,
             final AffinityTopologyVersion topVer,
             final IgniteTxLocalAdapter tx,
             final CacheOperationContext opCtx) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 72e1bb1cc3080..dc6d3dd8271dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -140,7 +140,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
     private CI2<GridNearAtomicAbstractUpdateRequest, GridNearAtomicUpdateResponse> updateReplyClos;
 
     /** Pending */
-    private GridDeferredAckMessageSender deferredUpdateMessageSender;
+    private GridDeferredAckMessageSender deferredUpdateMsgSnd;
 
     /** */
     private GridNearAtomicCache<K, V> near;
@@ -174,6 +174,11 @@ public GridDhtAtomicCache(GridCacheContext<K, V> ctx, GridCacheConcurrentMap map
         msgLog = ctx.shared().atomicMessageLogger();
     }
 
+    /** {@inheritDoc} */
+    @Override protected void checkJta() throws IgniteCheckedException {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isDhtAtomic() {
         return true;
@@ -236,7 +241,7 @@ else if (res.error() != null) {
     @Override public void start() throws IgniteCheckedException {
         super.start();
 
-        deferredUpdateMessageSender = new GridDeferredAckMessageSender(ctx.time(), ctx.closures()) {
+        deferredUpdateMsgSnd = new GridDeferredAckMessageSender(ctx.time(), ctx.closures()) {
             @Override public int getTimeout() {
                 return DEFERRED_UPDATE_RESPONSE_TIMEOUT;
             }
@@ -448,7 +453,7 @@ else if (res.error() != null) {
 
     /** {@inheritDoc} */
     @Override public void stop() {
-        deferredUpdateMessageSender.stop();
+        deferredUpdateMsgSnd.stop();
     }
 
     /**
@@ -464,7 +469,7 @@ public void near(GridNearAtomicCache<K, V> near) {
     }
 
     /** {@inheritDoc} */
-    @Override protected V get(K key, String taskName, boolean deserializeBinary, boolean needVer)
+    @Override protected V get0(K key, String taskName, boolean deserializeBinary, boolean needVer)
         throws IgniteCheckedException {
         ctx.checkSecurity(SecurityPermission.CACHE_READ);
 
@@ -540,6 +545,21 @@ public void near(GridNearAtomicCache<K, V> near) {
         });
     }
 
+    /** {@inheritDoc} */
+    @Override protected Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
+        throws IgniteCheckedException {
+        return getAllAsyncInternal(keys,
+            !ctx.config().isReadFromBackup(),
+            true,
+            null,
+            ctx.kernalContext().job().currentTaskName(),
+            deserializeBinary,
+            false,
+            true,
+            needVer,
+            false).get();
+    }
+
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<Map<K, V>> getAllAsync(
         @Nullable final Collection<? extends K> keys,
@@ -551,6 +571,43 @@ public void near(GridNearAtomicCache<K, V> near) {
         final boolean skipVals,
         final boolean canRemap,
         final boolean needVer
+    ) {
+        return getAllAsyncInternal(keys,
+            forcePrimary,
+            skipTx,
+            subjId,
+            taskName,
+            deserializeBinary,
+            skipVals,
+            canRemap,
+            needVer,
+            true);
+    }
+
+    /**
+     * @param keys Keys.
+     * @param forcePrimary Force primary flag.
+     * @param skipTx Skip tx flag.
+     * @param subjId Subject ID.
+     * @param taskName Task name.
+     * @param deserializeBinary Deserialize binary flag.
+     * @param skipVals Skip values flag.
+     * @param canRemap Can remap flag.
+     * @param needVer Need version flag.
+     * @param asyncOp Async operation flag.
+     * @return Future.
+     */
+    private IgniteInternalFuture<Map<K, V>> getAllAsyncInternal(
+        @Nullable final Collection<? extends K> keys,
+        final boolean forcePrimary,
+        boolean skipTx,
+        @Nullable UUID subjId,
+        final String taskName,
+        final boolean deserializeBinary,
+        final boolean skipVals,
+        final boolean canRemap,
+        final boolean needVer,
+        boolean asyncOp
     ) {
         ctx.checkSecurity(SecurityPermission.CACHE_READ);
 
@@ -562,7 +619,7 @@ public void near(GridNearAtomicCache<K, V> near) {
 
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
-        subjId = ctx.subjectIdPerCall(null, opCtx);
+        subjId = ctx.subjectIdPerCall(subjId, opCtx);
 
         final UUID subjId0 = subjId;
 
@@ -570,44 +627,77 @@ public void near(GridNearAtomicCache<K, V> near) {
 
         final boolean skipStore = opCtx != null && opCtx.skipStore();
 
-        return asyncOp(new CO<IgniteInternalFuture<Map<K, V>>>() {
-            @Override public IgniteInternalFuture<Map<K, V>> apply() {
-                return getAllAsync0(ctx.cacheKeysView(keys),
-                    forcePrimary,
-                    subjId0,
-                    taskName,
-                    deserializeBinary,
-                    expiryPlc,
-                    skipVals,
-                    skipStore,
-                    canRemap,
-                    needVer);
-            }
-        });
+        if (asyncOp) {
+            return asyncOp(new CO<IgniteInternalFuture<Map<K, V>>>() {
+                @Override public IgniteInternalFuture<Map<K, V>> apply() {
+                    return getAllAsync0(ctx.cacheKeysView(keys),
+                        forcePrimary,
+                        subjId0,
+                        taskName,
+                        deserializeBinary,
+                        expiryPlc,
+                        skipVals,
+                        skipStore,
+                        canRemap,
+                        needVer);
+                }
+            });
+        }
+        else {
+            return getAllAsync0(ctx.cacheKeysView(keys),
+                forcePrimary,
+                subjId0,
+                taskName,
+                deserializeBinary,
+                expiryPlc,
+                skipVals,
+                skipStore,
+                canRemap,
+                needVer);
+        }
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndPut(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
-        return getAndPutAsync0(key, val, filter).get();
+    @Override protected V getAndPut0(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
+        return (V)update0(
+            key,
+            val,
+            null,
+            null,
+            true,
+            filter,
+            true,
+            false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public boolean put(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
-        return putAsync(key, val, filter).get();
+    @Override protected boolean put0(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
+        Boolean res = (Boolean)update0(
+            key,
+            val,
+            null,
+            null,
+            false,
+            filter,
+            true,
+            false).get();
+
+        assert res != null;
+
+        return res;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<V> getAndPutAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
-        return updateAsync0(
+        return update0(
             key,
             val,
             null,
             null,
             true,
             filter,
+            true,
             true);
     }
 
@@ -616,13 +706,14 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
         A.notNull(key, "key", val, "val");
 
-        return updateAsync0(
+        return update0(
             key,
             val,
             null,
             null,
             false,
             filter,
+            true,
             true);
     }
 
@@ -630,84 +721,34 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public V tryGetAndPut(K key, V val) throws IgniteCheckedException {
         A.notNull(key, "key", val, "val");
 
-        return (V)updateAsync0(
+        return (V) update0(
             key,
             val,
             null,
             null,
             true,
             null,
+            false,
             false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
-        return getAndPutIfAbsentAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return getAndPutAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
-        return putIfAbsentAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return putAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
-        return getAndReplaceAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return getAndPutAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V val) throws IgniteCheckedException {
-        return replaceAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return putAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
-        return replaceAsync(key, oldVal, newVal).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
-        A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
-        return putAsync(key, newVal, ctx.equalsVal(oldVal));
-    }
-
-    /** {@inheritDoc} */
-    @Override public void putAll(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
-        putAllAsync(m).get();
+    @Override protected void putAll0(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        updateAll0(m,
+            null,
+            null,
+            null,
+            null,
+            false,
+            false,
+            true,
+            UPDATE,
+            false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> putAllAsync(Map<? extends K, ? extends V> m) {
-        return updateAllAsync0(m,
+    @Override public IgniteInternalFuture<?> putAllAsync0(Map<? extends K, ? extends V> m) {
+        return updateAll0(m,
             null,
             null,
             null,
@@ -715,7 +756,8 @@ public void near(GridNearAtomicCache<K, V> near) {
             false,
             false,
             true,
-            UPDATE).chain(RET2NULL);
+            UPDATE,
+            true).chain(RET2NULL);
     }
 
     /** {@inheritDoc} */
@@ -728,7 +770,7 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public IgniteInternalFuture<?> putAllConflictAsync(Map<KeyCacheObject, GridCacheDrInfo> conflictMap) {
         ctx.dr().onReceiveCacheEntriesReceived(conflictMap.size());
 
-        return updateAllAsync0(null,
+        return updateAll0(null,
             null,
             null,
             conflictMap,
@@ -736,57 +778,40 @@ public void near(GridNearAtomicCache<K, V> near) {
             false,
             false,
             true,
-            UPDATE);
+            UPDATE,
+            true);
     }
 
     /** {@inheritDoc} */
-    @Override public V getAndRemove(K key) throws IgniteCheckedException {
-        return getAndRemoveAsync(key).get();
+    @Override public V getAndRemove0(K key) throws IgniteCheckedException {
+        return (V)remove0(key, true, null, false).get();
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<V> getAndRemoveAsync(K key) {
-        A.notNull(key, "key");
-
-        return removeAsync0(key, true, null);
+    @Override public IgniteInternalFuture<V> getAndRemoveAsync0(K key) {
+        return remove0(key, true, null, true);
     }
 
     /** {@inheritDoc} */
-    @Override public void removeAll(Collection<? extends K> keys) throws IgniteCheckedException {
-        removeAllAsync(keys).get();
+    @Override protected void removeAll0(Collection<? extends K> keys) throws IgniteCheckedException {
+        removeAllAsync0(keys, null, false, false, false).get();
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> removeAllAsync(Collection<? extends K> keys) {
-        A.notNull(keys, "keys");
-
-        return removeAllAsync0(keys, null, false, false).chain(RET2NULL);
+    @Override public IgniteInternalFuture<Object> removeAllAsync0(Collection<? extends K> keys) {
+        return removeAllAsync0(keys, null, false, false, true).chain(RET2NULL);
     }
 
     /** {@inheritDoc} */
-    @Override public boolean remove(K key) throws IgniteCheckedException {
-        return removeAsync(key, (CacheEntryPredicate)null).get();
+    @Override protected boolean remove0(K key, CacheEntryPredicate filter) throws IgniteCheckedException {
+        return (Boolean)remove0(key, false, filter, false).get();
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key");
-
-        return removeAsync0(key, false, filter);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean remove(K key, V val) throws IgniteCheckedException {
-        return removeAsync(key, val).get();
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
-        A.notNull(key, "key", val, "val");
-
-        return removeAsync(key, ctx.equalsVal(val));
+    @Override public IgniteInternalFuture<Boolean> removeAsync0(K key, @Nullable CacheEntryPredicate filter) {
+        return remove0(key, false, filter, true);
     }
 
     /** {@inheritDoc} */
@@ -799,7 +824,7 @@ public void near(GridNearAtomicCache<K, V> near) {
     @Override public IgniteInternalFuture<?> removeAllConflictAsync(Map<KeyCacheObject, GridCacheVersion> conflictMap) {
         ctx.dr().onReceiveCacheEntriesReceived(conflictMap.size());
 
-        return removeAllAsync0(null, conflictMap, false, false);
+        return removeAllAsync0(null, conflictMap, false, false, true);
     }
 
     /**
@@ -814,7 +839,7 @@ private boolean writeThrough() {
      * @return Future.
      */
     @SuppressWarnings("unchecked")
-    protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>> op) {
+    private <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>> op) {
         IgniteInternalFuture<T> fail = asyncOpAcquire();
 
         if (fail != null)
@@ -872,7 +897,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     /** {@inheritDoc} */
     @Override public <T> EntryProcessorResult<T> invoke(K key, EntryProcessor<K, V, T> entryProcessor, Object... args)
         throws IgniteCheckedException {
-        IgniteInternalFuture<EntryProcessorResult<T>> invokeFut = invokeAsync(key, entryProcessor, args);
+        IgniteInternalFuture<EntryProcessorResult<T>> invokeFut = invoke0(false, key, entryProcessor, args);
 
         EntryProcessorResult<T> res = invokeFut.get();
 
@@ -882,14 +907,28 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     /** {@inheritDoc} */
     @Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(Set<? extends K> keys,
         EntryProcessor<K, V, T> entryProcessor,
-        Object... args)
-        throws IgniteCheckedException {
-        return invokeAllAsync(keys, entryProcessor, args).get();
+        Object... args) throws IgniteCheckedException
+    {
+        return invokeAll0(false, keys, entryProcessor, args).get();
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
     @Override public <T> IgniteInternalFuture<EntryProcessorResult<T>> invokeAsync(K key,
+        EntryProcessor<K, V, T> entryProcessor,
+        Object... args) {
+        return invoke0(true, key, entryProcessor, args);
+    }
+
+    /**
+     * @param async Async operation flag.
+     * @param key Key.
+     * @param entryProcessor Entry processor.
+     * @param args Entry processor arguments.
+     * @return Future.
+     */
+    private <T> IgniteInternalFuture<EntryProcessorResult<T>> invoke0(
+        boolean async,
+        K key,
         EntryProcessor<K, V, T> entryProcessor,
         Object... args) {
         A.notNull(key, "key", entryProcessor, "entryProcessor");
@@ -901,14 +940,15 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut = updateAsync0(
+        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut = update0(
             key,
             null,
             entryProcessor,
             args,
             false,
             null,
-            true);
+            true,
+            async);
 
         return fut.chain(new CX1<IgniteInternalFuture<Map<K, EntryProcessorResult<T>>>, EntryProcessorResult<T>>() {
             @Override public EntryProcessorResult<T> applyx(IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut)
@@ -939,6 +979,21 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public <T> IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> invokeAllAsync(Set<? extends K> keys,
+        final EntryProcessor<K, V, T> entryProcessor,
+        Object... args) {
+        return invokeAll0(true, keys, entryProcessor, args);
+    }
+
+    /**
+     * @param async Async operation flag.
+     * @param keys Keys.
+     * @param entryProcessor Entry processor.
+     * @param args Entry processor arguments.
+     * @return Future.
+     */
+    private <T> IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> invokeAll0(
+        boolean async,
+        Set<? extends K> keys,
         final EntryProcessor<K, V, T> entryProcessor,
         Object... args) {
         A.notNull(keys, "keys", entryProcessor, "entryProcessor");
@@ -956,7 +1011,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> resFut = updateAllAsync0(null,
+        IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> resFut = updateAll0(null,
             invokeMap,
             args,
             null,
@@ -964,7 +1019,8 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
             false,
             false,
             true,
-            TRANSFORM);
+            TRANSFORM,
+            async);
 
         return resFut.chain(
             new CX1<IgniteInternalFuture<Map<K, EntryProcessorResult<T>>>, Map<K, EntryProcessorResult<T>>>() {
@@ -982,7 +1038,21 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
     @Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(
         Map<? extends K, ? extends EntryProcessor<K, V, T>> map,
         Object... args) throws IgniteCheckedException {
-        return invokeAllAsync(map, args).get();
+        A.notNull(map, "map");
+
+        if (keyCheck)
+            validateCacheKeys(map.keySet());
+
+        return (Map<K, EntryProcessorResult<T>>)updateAll0(null,
+            map,
+            args,
+            null,
+            null,
+            false,
+            false,
+            true,
+            TRANSFORM,
+            false).get();
     }
 
     /** {@inheritDoc} */
@@ -995,7 +1065,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
         if (keyCheck)
             validateCacheKeys(map.keySet());
 
-        return updateAllAsync0(null,
+        return updateAll0(null,
             map,
             args,
             null,
@@ -1003,7 +1073,8 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
             false,
             false,
             true,
-            TRANSFORM);
+            TRANSFORM,
+            true);
     }
 
     /**
@@ -1017,10 +1088,11 @@ protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>>
      * @param retval Return value required flag.
      * @param rawRetval Return {@code GridCacheReturn} instance.
      * @param waitTopFut Whether to wait for topology future.
+     * @param async Async operation flag.
      * @return Completion future.
      */
     @SuppressWarnings("ConstantConditions")
-    private IgniteInternalFuture updateAllAsync0(
+    private IgniteInternalFuture updateAll0(
         @Nullable Map<? extends K, ? extends V> map,
         @Nullable Map<? extends K, ? extends EntryProcessor> invokeMap,
         @Nullable Object[] invokeArgs,
@@ -1029,7 +1101,8 @@ private IgniteInternalFuture updateAllAsync0(
         final boolean retval,
         final boolean rawRetval,
         final boolean waitTopFut,
-        final GridCacheOperation op
+        final GridCacheOperation op,
+        boolean async
     ) {
         assert ctx.updatesAllowed();
 
@@ -1106,13 +1179,20 @@ else if (op == GridCacheOperation.DELETE) {
             opCtx != null && opCtx.noRetries() ? 1 : MAX_RETRIES,
             waitTopFut);
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-                return updateFut;
-            }
-        });
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
+
+            return updateFut;
+        }
     }
 
     /**
@@ -1125,16 +1205,18 @@ else if (op == GridCacheOperation.DELETE) {
      * @param retval Return value flag.
      * @param filter Filter.
      * @param waitTopFut Whether to wait for topology future.
+     * @param async Async operation flag.
      * @return Future.
      */
-    private IgniteInternalFuture updateAsync0(
+    private IgniteInternalFuture update0(
         K key,
         @Nullable V val,
         @Nullable EntryProcessor proc,
         @Nullable Object[] invokeArgs,
         final boolean retval,
         @Nullable final CacheEntryPredicate filter,
-        final boolean waitTopFut
+        final boolean waitTopFut,
+        boolean async
     ) {
         assert val == null || proc == null;
 
@@ -1147,13 +1229,20 @@ private IgniteInternalFuture updateAsync0(
         final GridNearAtomicAbstractUpdateFuture updateFut =
             createSingleUpdateFuture(key, val, proc, invokeArgs, retval, filter, waitTopFut);
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-                return updateFut;
-            }
-        });
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
+
+            return updateFut;
+        }
     }
 
     /**
@@ -1162,33 +1251,38 @@ private IgniteInternalFuture updateAsync0(
      * @param key Key.
      * @param retval Whether to return
      * @param filter Filter.
+     * @param async Async operation flag.
      * @return Future.
      */
-    private IgniteInternalFuture removeAsync0(K key, final boolean retval,
-        @Nullable CacheEntryPredicate filter) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
+    private IgniteInternalFuture remove0(K key, final boolean retval,
+        @Nullable CacheEntryPredicate filter,
+        boolean async) {
         assert ctx.updatesAllowed();
 
-        validateCacheKey(key);
-
         ctx.checkSecurity(SecurityPermission.CACHE_REMOVE);
 
-        final GridNearAtomicAbstractUpdateFuture updateFut =
-            createSingleUpdateFuture(key, null, null, null, retval, filter, true);
+        final GridNearAtomicAbstractUpdateFuture updateFut = createSingleUpdateFuture(key,
+            null,
+            null,
+            null,
+            retval,
+            filter,
+            true);
 
-        if (statsEnabled)
-            updateFut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
 
-                return updateFut;
-            }
-        });
+            return updateFut;
+        }
     }
 
     /**
@@ -1327,14 +1421,11 @@ private IgniteInternalFuture removeAllAsync0(
         @Nullable Collection<? extends K> keys,
         @Nullable Map<KeyCacheObject, GridCacheVersion> conflictMap,
         final boolean retval,
-        boolean rawRetval
+        boolean rawRetval,
+        boolean async
     ) {
         assert ctx.updatesAllowed();
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         assert keys != null || conflictMap != null;
 
         if (keyCheck)
@@ -1381,16 +1472,20 @@ private IgniteInternalFuture removeAllAsync0(
             opCtx != null && opCtx.noRetries() ? 1 : MAX_RETRIES,
             true);
 
-        if (statsEnabled)
-            updateFut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+        if (async) {
+            return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+                @Override public IgniteInternalFuture<Object> apply() {
+                    updateFut.map();
 
-        return asyncOp(new CO<IgniteInternalFuture<Object>>() {
-            @Override public IgniteInternalFuture<Object> apply() {
-                updateFut.map();
+                    return updateFut;
+                }
+            });
+        }
+        else {
+            updateFut.map();
 
-                return updateFut;
-            }
-        });
+            return updateFut;
+        }
     }
 
     /**
@@ -3252,7 +3347,7 @@ && writeThrough() && !req.skipStore(),
      * @param ver Version to ack.
      */
     private void sendDeferredUpdateResponse(UUID nodeId, GridCacheVersion ver) {
-        deferredUpdateMessageSender.sendDeferredAckMessage(nodeId, ver);
+        deferredUpdateMsgSnd.sendDeferredAckMessage(nodeId, ver);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index ccdc51d2f6139..c9fc983d4850c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -328,19 +328,6 @@ public GridDistributedCacheEntry entryExx(
             needVer);
     }
 
-    /** {@inheritDoc} */
-    @Override protected GridCacheEntryEx entryExSafe(
-        KeyCacheObject key,
-        AffinityTopologyVersion topVer
-    ) {
-        try {
-            return ctx.affinity().localNode(key, topVer) ? entryEx(key) : null;
-        }
-        catch (GridDhtInvalidPartitionException ignored) {
-            return null;
-        }
-    }
-
     /**
      * @param keys Keys to load.
      * @param readThrough Read through flag.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index d1056fdff4284..b843e4e6d3db1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -449,61 +449,11 @@ public void processDhtAtomicUpdateRequest(
         return dht.putAsync0(key, val, filter);
     }
 
-    /** {@inheritDoc} */
-    @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
-        return dht.getAndPutIfAbsent(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
-        return dht.getAndPutIfAbsentAsync(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
-        return dht.putIfAbsent(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
-        return dht.putIfAbsentAsync(key, val);
-    }
-
     /** {@inheritDoc} */
     @Nullable @Override public V tryGetAndPut(K key, V val) throws IgniteCheckedException {
         return dht.tryGetAndPut(key, val);
     }
 
-    /** {@inheritDoc} */
-    @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
-        return dht.getAndReplace(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
-        return dht.getAndReplaceAsync(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V val) throws IgniteCheckedException {
-        return dht.replace(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
-        return dht.replaceAsync(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
-        return dht.replace(key, oldVal, newVal);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
-        return dht.replaceAsync(key, oldVal, newVal);
-    }
-
     /** {@inheritDoc} */
     @Override public void putAll(Map<? extends K, ? extends V> m)
         throws IgniteCheckedException {
@@ -568,6 +518,11 @@ public void processDhtAtomicUpdateRequest(
         return dht.invokeAllAsync(keys, entryProcessor, args);
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean remove(K key, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
+        return dht.remove(key, filter);
+    }
+
     /** {@inheritDoc} */
     @Override public V getAndRemove(K key) throws IgniteCheckedException {
         return dht.getAndRemove(key);
@@ -601,16 +556,6 @@ public void processDhtAtomicUpdateRequest(
         return dht.removeAsync(key, filter);
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean remove(K key, V val) throws IgniteCheckedException {
-        return dht.remove(key, val);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
-        return dht.removeAsync(key, val);
-    }
-
     /** {@inheritDoc} */
     @Override public void removeAll() throws IgniteCheckedException {
         dht.removeAll();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index 7da11b6b870be..f86df2f29d684 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -107,6 +107,11 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
         preldr = new GridCachePreloaderAdapter(ctx);
     }
 
+    /** {@inheritDoc} */
+    @Override protected void checkJta() throws IgniteCheckedException {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isLocal() {
         return true;
@@ -119,9 +124,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public V getAndPut(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
+    @Override protected V getAndPut0(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         return (V)updateAllInternal(UPDATE,
@@ -138,16 +141,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean put(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
+    @Override protected boolean put0(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
-        boolean res = (Boolean)updateAllInternal(UPDATE,
+        Boolean res = (Boolean)updateAllInternal(UPDATE,
             Collections.singleton(key),
             Collections.singleton(val),
             null,
@@ -159,8 +156,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
             ctx.readThrough(),
             opCtx != null && opCtx.isKeepBinary());
 
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
+        assert res != null;
 
         return res;
     }
@@ -168,8 +164,6 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<V> getAndPutAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
         return updateAllAsync0(F0.asMap(key, val),
             null,
             null,
@@ -181,8 +175,6 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key", val, "val");
-
         return updateAllAsync0(F0.asMap(key, val),
             null,
             null,
@@ -192,65 +184,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
-        return getAndPut(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
-        return getAndPutAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
-        return put(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
-        return putAsync(key, val, ctx.noVal());
-    }
-
-    /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
-        return getAndPut(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
-        return getAndPutAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V val) throws IgniteCheckedException {
-        return put(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
-        return putAsync(key, val, ctx.hasVal());
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
-        A.notNull(oldVal, "oldVal");
-
-        return put(key, newVal, ctx.equalsVal(oldVal));
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
-        return putAsync(key, newVal, ctx.equalsVal(oldVal));
-    }
-
-    /** {@inheritDoc} */
-    @Override public void putAll(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
+    @Override protected void putAll0(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         updateAllInternal(UPDATE,
@@ -264,13 +198,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
             ctx.writeThrough(),
             ctx.readThrough(),
             opCtx != null && opCtx.isKeepBinary());
-
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> putAllAsync(Map<? extends K, ? extends V> m) {
+    @Override public IgniteInternalFuture<?> putAllAsync0(Map<? extends K, ? extends V> m) {
         return updateAllAsync0(m,
             null,
             null,
@@ -280,8 +211,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
-    @Override public V getAndRemove(K key) throws IgniteCheckedException {
+    @Override protected V getAndRemove0(K key) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         return (V)updateAllInternal(DELETE,
@@ -299,13 +229,13 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<V> getAndRemoveAsync(K key) {
+    @Override public IgniteInternalFuture<V> getAndRemoveAsync0(K key) {
         return removeAllAsync0(Collections.singletonList(key), true, false, null);
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public void removeAll(Collection<? extends K> keys) throws IgniteCheckedException {
+    @Override public void removeAll0(Collection<? extends K> keys) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         updateAllInternal(DELETE,
@@ -322,19 +252,13 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<?> removeAllAsync(Collection<? extends K> keys) {
+    @Override public IgniteInternalFuture<Object> removeAllAsync0(Collection<? extends K> keys) {
         return removeAllAsync0(keys, false, false, null).chain(RET2NULL);
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public boolean remove(K key) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
-        A.notNull(key, "key");
-
+    @Override public boolean remove0(K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         Boolean rmv = (Boolean)updateAllInternal(DELETE,
@@ -344,49 +268,22 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
             expiryPerCall(),
             false,
             false,
-            null,
+            filter,
             ctx.writeThrough(),
             ctx.readThrough(),
             opCtx != null && opCtx.isKeepBinary());
 
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+        assert rmv != null;
 
         return rmv;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, @Nullable CacheEntryPredicate filter) {
-        A.notNull(key, "key");
-
+    @Override public IgniteInternalFuture<Boolean> removeAsync0(K key, @Nullable CacheEntryPredicate filter) {
         return removeAllAsync0(Collections.singletonList(key), false, false, filter);
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean remove(K key, V val) throws IgniteCheckedException {
-        A.notNull(key, "key", val, "val");
-
-        CacheOperationContext opCtx = ctx.operationContextPerCall();
-
-        return (Boolean)updateAllInternal(DELETE,
-            Collections.singleton(key),
-            null,
-            null,
-            expiryPerCall(),
-            false,
-            false,
-            ctx.equalsVal(val),
-            ctx.writeThrough(),
-            ctx.readThrough(),
-            opCtx != null && opCtx.isKeepBinary());
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
-        return removeAsync(key, ctx.equalsVal(val));
-    }
-
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<?> removeAllAsync() {
         return ctx.closures().callLocalSafe(new Callable<Void>() {
@@ -399,11 +296,13 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     }
 
     /** {@inheritDoc} */
-
     @SuppressWarnings("unchecked")
-    @Override @Nullable public V get(K key, boolean deserializeBinary, boolean needVer) throws IgniteCheckedException {
-        String taskName = ctx.kernalContext().job().currentTaskName();
-
+    @Override protected V get0(
+        final K key,
+        String taskName,
+        boolean deserializeBinary,
+        boolean needVer) throws IgniteCheckedException
+    {
         Map<K, V> m = getAllInternal(Collections.singleton(key),
             ctx.isSwapOrOffheapEnabled(),
             ctx.readThrough(),
@@ -419,7 +318,7 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public final Map<K, V> getAll(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
+    @Override public final Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
         throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
@@ -795,7 +694,7 @@ private IgniteInternalFuture updateAllAsync0(
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture fut = asyncOp(new Callable<Object>() {
+        return asyncOp(new Callable<Object>() {
             @Override public Object call() throws Exception {
                 return updateAllInternal(op,
                     keys,
@@ -810,11 +709,6 @@ private IgniteInternalFuture updateAllAsync0(
                     keepBinary);
             }
         });
-
-        if (ctx.config().isStatisticsEnabled())
-            fut.listen(new UpdatePutTimeStatClosure(metrics0(), System.nanoTime()));
-
-        return fut;
     }
 
     /**
@@ -836,17 +730,13 @@ private IgniteInternalFuture removeAllAsync0(
 
         final boolean readThrough = ctx.readThrough();
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         final ExpiryPolicy expiryPlc = expiryPerCall();
 
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
 
-        IgniteInternalFuture fut = asyncOp(new Callable<Object>() {
+        return asyncOp(new Callable<Object>() {
             @Override public Object call() throws Exception {
                 return updateAllInternal(DELETE,
                     keys,
@@ -861,11 +751,6 @@ private IgniteInternalFuture removeAllAsync0(
                     keepBinary);
             }
         });
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
-
-        return fut;
     }
 
     /**
@@ -1586,7 +1471,7 @@ private static CachePartialUpdateCheckedException partialUpdateException() {
      * @return Future.
      */
     @SuppressWarnings("unchecked")
-    protected IgniteInternalFuture asyncOp(final Callable<?> op) {
+    private IgniteInternalFuture asyncOp(final Callable<?> op) {
         IgniteInternalFuture fail = asyncOpAcquire();
 
         if (fail != null)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
index 3e3b84e4403bf..648134e498c85 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
@@ -34,7 +34,7 @@ public class CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest extend
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
-        cfg.getTransactionConfiguration().setDefaultTxTimeout(Long.MAX_VALUE);
+        cfg.getTransactionConfiguration().setDefaultTxTimeout(5 * 60_000);
 
         return cfg;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
index ec3b8089cae6e..a235d8f15e3f6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteFileSystem;
@@ -39,14 +41,11 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.transactions.TransactionConcurrency;
-import org.apache.ignite.transactions.TransactionIsolation;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
 /**
  * Test to check for system pool starvation due to {@link IgfsBlocksMessage}.
@@ -125,8 +124,7 @@ public void testStarvation() throws Exception {
             @Override public Void call() throws Exception {
                 GridCacheAdapter dataCache = dataCache(attacker);
 
-                try (IgniteInternalTx tx =
-                         dataCache.txStartEx(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+                try (IgniteInternalTx tx = dataCache.txStartEx(PESSIMISTIC, REPEATABLE_READ)) {
                     dataCache.put(DATA_KEY, 0);
 
                     txStartLatch.countDown();
@@ -185,6 +183,7 @@ private static boolean awaitFuture(final IgniteInternalFuture fut) throws Except
      * Create IGFS file asynchronously.
      *
      * @param path Path.
+     * @param writeStartLatch Write start latch.
      * @return Future.
      */
     private IgniteInternalFuture<Void> createFileAsync(final IgfsPath path, final CountDownLatch writeStartLatch) {

From 0fc6271d8e39125bf5ee341e50a2665a04fc8b1e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Jun 2017 13:42:12 +0300
Subject: [PATCH 222/516] GG-12350: GridDhtAtomicSingleUpdateRequest misses
 topologyVersion() method override.

---
 .../dht/atomic/GridDhtAtomicSingleUpdateRequest.java         | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
index a03d948017f91..0838cb6dd87b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
@@ -360,6 +360,11 @@ public GridDhtAtomicSingleUpdateRequest() {
         return isFlag(KEEP_BINARY_FLAG_MASK);
     }
 
+    /** {@inheritDoc} */
+    @Override public AffinityTopologyVersion topologyVersion() {
+        return topVer;
+    }
+
     /**
      *
      */

From 113a1380da34ea804d68757d39926da97dee09b6 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 13 Jun 2017 08:20:22 +0300
Subject: [PATCH 223/516] GG-12355: Backported IO latency test.

---
 .../org/apache/ignite/internal/GridTopic.java |   5 +-
 .../apache/ignite/internal/IgniteKernal.java  |  16 +-
 .../managers/communication/GridIoManager.java | 699 +++++++++++++++++-
 .../communication/GridIoMessageFactory.java   |   5 +
 .../communication/IgniteIoTestMessage.java    | 594 +++++++++++++++
 .../apache/ignite/mxbean/IgniteMXBean.java    |  44 ++
 6 files changed, 1358 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 248f75b9d2689..dc20be002a6e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -94,7 +94,10 @@ public enum GridTopic {
     TOPIC_QUERY,
 
     /** */
-    TOPIC_TX;
+    TOPIC_TX,
+
+    /** */
+    TOPIC_IO_TEST;
 
     /** Enum values. */
     private static final GridTopic[] VALS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 25f7884c889b9..4237aeebe49ce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -3545,7 +3545,7 @@ private boolean skipDaemon(GridComponent comp) {
     }
 
     /** {@inheritDoc} */
-    public void dumpDebugInfo() {
+    @Override public void dumpDebugInfo() {
         try {
             GridKernalContextImpl ctx = this.ctx;
 
@@ -3615,6 +3615,20 @@ void waitPreviousReconnect() {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void runIoTest(
+        long warmup,
+        long duration,
+        int threads,
+        long maxLatency,
+        int rangesCnt,
+        int payLoadSize,
+        boolean procFromNioThread
+    ) {
+        ctx.io().runIoTest(warmup, duration, threads, maxLatency, rangesCnt, payLoadSize, procFromNioThread,
+            new ArrayList(ctx.cluster().get().forServers().forRemotes().nodes()));
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IgniteKernal.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 3df29cf8fb4e5..4b68e5bdd0da7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -18,23 +18,33 @@
 package org.apache.ignite.internal.managers.communication;
 
 import java.io.Serializable;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Queue;
 import java.util.UUID;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
@@ -44,6 +54,7 @@
 import org.apache.ignite.internal.GridTopic;
 import org.apache.ignite.internal.IgniteComponentType;
 import org.apache.ignite.internal.IgniteDeploymentCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.direct.DirectMessageReader;
 import org.apache.ignite.internal.direct.DirectMessageWriter;
 import org.apache.ignite.internal.managers.GridManagerAdapter;
@@ -55,7 +66,10 @@
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet;
 import org.apache.ignite.internal.util.GridSpinReadWriteLock;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridTuple3;
+import org.apache.ignite.internal.util.lang.IgnitePair;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
@@ -78,11 +92,13 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 import org.jsr166.ConcurrentLinkedDeque8;
+import org.jsr166.LongAdder8;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.GridTopic.TOPIC_COMM_USER;
+import static org.apache.ignite.internal.GridTopic.TOPIC_IO_TEST;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.AFFINITY_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.IDX_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.IGFS_POOL;
@@ -176,6 +192,12 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
     /** Stopping flag. */
     private boolean stopping;
 
+    /** */
+    private final AtomicReference<ConcurrentHashMap<Long, IoTestFuture>> ioTestMap = new AtomicReference<>();
+
+    /** */
+    private final AtomicLong ioTestId = new AtomicLong();
+
     /**
      * @param ctx Grid kernal context.
      */
@@ -297,6 +319,399 @@ public void resetMetrics() {
 
         if (log.isDebugEnabled())
             log.debug(startInfo());
+
+        addMessageListener(TOPIC_IO_TEST, new GridMessageListener() {
+            @Override public void onMessage(UUID nodeId, Object msg) {
+                ClusterNode node = ctx.discovery().node(nodeId);
+
+                if (node == null)
+                    return;
+
+                IgniteIoTestMessage msg0 = (IgniteIoTestMessage)msg;
+
+                msg0.senderNodeId(nodeId);
+
+                if (msg0.request()) {
+                    IgniteIoTestMessage res = new IgniteIoTestMessage(msg0.id(), false, null);
+
+                    res.flags(msg0.flags());
+                    res.onRequestProcessed();
+
+                    res.copyDataFromRequest(msg0);
+
+                    try {
+                        send(node, TOPIC_IO_TEST, res, GridIoPolicy.SYSTEM_POOL);
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to send IO test response [msg=" + msg0 + "]", e);
+                    }
+                }
+                else {
+                    IoTestFuture fut = ioTestMap().get(msg0.id());
+
+                    msg0.onResponseProcessed();
+
+                    if (fut == null)
+                        U.warn(log, "Failed to find IO test future [msg=" + msg0 + ']');
+                    else
+                        fut.onResponse(msg0);
+                }
+            }
+        });
+    }
+
+    /**
+     * @param nodes Nodes.
+     * @param payload Payload.
+     * @param procFromNioThread If {@code true} message is processed from NIO thread.
+     * @return Response future.
+     */
+    public IgniteInternalFuture sendIoTest(List<ClusterNode> nodes, byte[] payload, boolean procFromNioThread) {
+        long id = ioTestId.getAndIncrement();
+
+        IoTestFuture fut = new IoTestFuture(id, nodes.size());
+
+        IgniteIoTestMessage msg = new IgniteIoTestMessage(id, true, payload);
+
+        msg.processFromNioThread(procFromNioThread);
+
+        ioTestMap().put(id, fut);
+
+        for (int i = 0; i < nodes.size(); i++) {
+            ClusterNode node = nodes.get(i);
+
+            try {
+                send(node, TOPIC_IO_TEST, msg, GridIoPolicy.SYSTEM_POOL);
+            }
+            catch (IgniteCheckedException e) {
+                ioTestMap().remove(msg.id());
+
+                return new GridFinishedFuture(e);
+            }
+        }
+
+        return fut;
+    }
+
+    /**
+     * @param node Node.
+     * @param payload Payload.
+     * @param procFromNioThread If {@code true} message is processed from NIO thread.
+     * @return Response future.
+     */
+    public IgniteInternalFuture<List<IgniteIoTestMessage>> sendIoTest(
+        ClusterNode node,
+        byte[] payload,
+        boolean procFromNioThread
+    ) {
+        long id = ioTestId.getAndIncrement();
+
+        IoTestFuture fut = new IoTestFuture(id, 1);
+
+        IgniteIoTestMessage msg = new IgniteIoTestMessage(id, true, payload);
+
+        msg.processFromNioThread(procFromNioThread);
+
+        ioTestMap().put(id, fut);
+
+        try {
+            send(node, TOPIC_IO_TEST, msg, GridIoPolicy.SYSTEM_POOL);
+        }
+        catch (IgniteCheckedException e) {
+            ioTestMap().remove(msg.id());
+
+            return new GridFinishedFuture(e);
+        }
+
+        return fut;
+    }
+
+    /**
+     * @return IO test futures map.
+     */
+    private ConcurrentHashMap<Long, IoTestFuture> ioTestMap() {
+        ConcurrentHashMap<Long, IoTestFuture> map = ioTestMap.get();
+
+        if (map == null) {
+            if (!ioTestMap.compareAndSet(null, map = new ConcurrentHashMap<>()))
+                map = ioTestMap.get();
+        }
+
+        return map;
+    }
+
+    /**
+     * @param warmup Warmup duration in milliseconds.
+     * @param duration Test duration in milliseconds.
+     * @param threads Thread count.
+     * @param latencyLimit Max latency in nanoseconds.
+     * @param rangesCnt Ranges count in resulting histogram.
+     * @param payLoadSize Payload size in bytes.
+     * @param procFromNioThread {@code True} to process requests in NIO threads.
+     * @param nodes Nodes participating in test.
+     */
+    public void runIoTest(
+        final long warmup,
+        final long duration,
+        final int threads,
+        final long latencyLimit,
+        final int rangesCnt,
+        final int payLoadSize,
+        final boolean procFromNioThread,
+        final List<ClusterNode> nodes
+    ) {
+        final ExecutorService svc = Executors.newFixedThreadPool(threads + 1);
+
+        final AtomicBoolean warmupFinished = new AtomicBoolean();
+        final AtomicBoolean done = new AtomicBoolean();
+        final CyclicBarrier bar = new CyclicBarrier(threads + 1);
+        final LongAdder8 cnt = new LongAdder8();
+        final long sleepDuration = 5000;
+        final byte[] payLoad = new byte[payLoadSize];
+        final Map<UUID, IoTestThreadLocalNodeResults>[] res = new Map[threads];
+
+        boolean failed = true;
+
+        try {
+            svc.execute(new Runnable() {
+                @Override public void run() {
+                    boolean failed = true;
+
+                    try {
+                        bar.await();
+
+                        long start = System.currentTimeMillis();
+
+                        if (log.isInfoEnabled())
+                            log.info("IO test started " +
+                                "[warmup=" + warmup +
+                                ", duration=" + duration +
+                                ", threads=" + threads +
+                                ", latencyLimit=" + latencyLimit +
+                                ", rangesCnt=" + rangesCnt +
+                                ", payLoadSize=" + payLoadSize +
+                                ", procFromNioThreads=" + procFromNioThread + ']'
+                            );
+
+                        for (;;) {
+                            if (!warmupFinished.get() && System.currentTimeMillis() - start > warmup) {
+                                if (log.isInfoEnabled())
+                                    log.info("IO test warmup finished.");
+
+                                warmupFinished.set(true);
+
+                                start = System.currentTimeMillis();
+                            }
+
+                            if (warmupFinished.get() && System.currentTimeMillis() - start > duration) {
+                                if (log.isInfoEnabled())
+                                    log.info("IO test finished, will wait for all threads to finish.");
+
+                                done.set(true);
+
+                                bar.await();
+
+                                failed = false;
+
+                                break;
+                            }
+
+                            if (log.isInfoEnabled())
+                                log.info("IO test [opsCnt/sec=" + (cnt.sumThenReset() * 1000 / sleepDuration) +
+                                    ", warmup=" + !warmupFinished.get() +
+                                    ", elapsed=" + (System.currentTimeMillis() - start) + ']');
+
+                            Thread.sleep(sleepDuration);
+                        }
+
+                        // At this point all threads have finished the test and
+                        // stored data to the resulting array of maps.
+                        // Need to iterate it over and sum values for all threads.
+                        printIoTestResults(res);
+                    }
+                    catch (InterruptedException | BrokenBarrierException e) {
+                        U.error(log, "IO test failed.", e);
+                    }
+                    finally {
+                        if (failed)
+                            bar.reset();
+
+                        svc.shutdown();
+                    }
+                }
+            });
+
+            for (int i = 0; i < threads; i++) {
+                final int i0 = i;
+
+                res[i] = U.newHashMap(nodes.size());
+
+                svc.execute(new Runnable() {
+                    @Override public void run() {
+                        boolean failed = true;
+                        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+                        int size = nodes.size();
+                        Map<UUID, IoTestThreadLocalNodeResults> res0 = res[i0];
+
+                        try {
+                            boolean warmupFinished0 = false;
+
+                            bar.await();
+
+                            for (;;) {
+                                if (done.get())
+                                    break;
+
+                                if (!warmupFinished0)
+                                    warmupFinished0 = warmupFinished.get();
+
+                                ClusterNode node = nodes.get(rnd.nextInt(size));
+
+                                List<IgniteIoTestMessage> msgs = sendIoTest(node, payLoad, procFromNioThread).get();
+
+                                cnt.increment();
+
+                                for (IgniteIoTestMessage msg : msgs) {
+                                    UUID nodeId = msg.senderNodeId();
+
+                                    assert nodeId != null;
+
+                                    IoTestThreadLocalNodeResults nodeRes = res0.get(nodeId);
+
+                                    if (nodeRes == null)
+                                        res0.put(nodeId,
+                                            nodeRes = new IoTestThreadLocalNodeResults(rangesCnt, latencyLimit));
+
+                                    nodeRes.onResult(msg);
+                                }
+                            }
+
+                            bar.await();
+
+                            failed = false;
+                        }
+                        catch (Exception e) {
+                            U.error(log, "IO test worker thread failed.", e);
+                        }
+                        finally {
+                            if (failed)
+                                bar.reset();
+                        }
+                    }
+                });
+            }
+
+            failed = false;
+        }
+        finally {
+            if (failed)
+                U.shutdownNow(GridIoManager.class, svc, log);
+        }
+    }
+
+    /**
+     * @param rawRes Resulting map.
+     */
+    private void printIoTestResults(
+        Map<UUID, IoTestThreadLocalNodeResults>[] rawRes
+    ) {
+        Map<UUID, IoTestNodeResults> res = new HashMap<>();
+
+        for (Map<UUID, IoTestThreadLocalNodeResults> r : rawRes) {
+            for (Entry<UUID, IoTestThreadLocalNodeResults> e : r.entrySet()) {
+                IoTestNodeResults r0 = res.get(e.getKey());
+
+                if (r0 == null)
+                    res.put(e.getKey(), r0 = new IoTestNodeResults());
+
+                r0.add(e.getValue());
+            }
+        }
+
+        SimpleDateFormat dateFmt = new SimpleDateFormat("HH:mm:ss,SSS");
+
+        StringBuilder b = new StringBuilder(U.nl())
+            .append("IO test results (round-trip count per each latency bin).")
+            .append(U.nl());
+
+        for (Entry<UUID, IoTestNodeResults> e : res.entrySet()) {
+            ClusterNode node = ctx.discovery().node(e.getKey());
+
+            long binLatencyMcs = e.getValue().binLatencyMcs();
+
+            b.append("Node ID: ").append(e.getKey()).append(" (addrs=")
+                .append(node != null ? node.addresses().toString() : "n/a")
+                .append(", binLatency=").append(binLatencyMcs).append("mcs")
+                .append(')').append(U.nl());
+
+            b.append("Latency bin, mcs | Count exclusive | Percentage exclusive | " +
+                "Count inclusive | Percentage inclusive ").append(U.nl());
+
+            long[] nodeRes = e.getValue().resLatency;
+
+            long sum = 0;
+
+            for (int i = 0; i < nodeRes.length; i++)
+                sum += nodeRes[i];
+
+            long curSum = 0;
+
+            for (int i = 0; i < nodeRes.length; i++) {
+                curSum += nodeRes[i];
+
+                if (i < nodeRes.length - 1)
+                    b.append(String.format("<%11d mcs | %15d | %19.6f%% | %15d | %19.6f%%\n",
+                        (i + 1) * binLatencyMcs,
+                        nodeRes[i], (100.0 * nodeRes[i]) / sum,
+                        curSum, (100.0 * curSum) / sum));
+                else
+                    b.append(String.format(">%11d mcs | %15d | %19.6f%% | %15d | %19.6f%%\n",
+                        i * binLatencyMcs,
+                        nodeRes[i], (100.0 * nodeRes[i]) / sum,
+                        curSum, (100.0 * curSum) / sum));
+            }
+
+            b.append(U.nl()).append("Total latency (ns): ").append(U.nl())
+                .append(String.format("%15d", e.getValue().totalLatency)).append(U.nl());
+
+            b.append(U.nl()).append("Max latencies (ns):").append(U.nl());
+            format(b, e.getValue().maxLatency, dateFmt);
+
+            b.append(U.nl()).append("Max request send queue times (ns):").append(U.nl());
+            format(b, e.getValue().maxReqSendQueueTime, dateFmt);
+
+            b.append(U.nl()).append("Max request receive queue times (ns):").append(U.nl());
+            format(b, e.getValue().maxReqRcvQueueTime, dateFmt);
+
+            b.append(U.nl()).append("Max response send queue times (ns):").append(U.nl());
+            format(b, e.getValue().maxResSendQueueTime, dateFmt);
+
+            b.append(U.nl()).append("Max response receive queue times (ns):").append(U.nl());
+            format(b, e.getValue().maxResRcvQueueTime, dateFmt);
+
+            b.append(U.nl()).append("Max request wire times (millis):").append(U.nl());
+            format(b, e.getValue().maxReqWireTimeMillis, dateFmt);
+
+            b.append(U.nl()).append("Max response wire times (millis):").append(U.nl());
+            format(b, e.getValue().maxResWireTimeMillis, dateFmt);
+
+            b.append(U.nl());
+        }
+
+        if (log.isInfoEnabled())
+            log.info(b.toString());
+    }
+
+    /**
+     * @param b Builder.
+     * @param pairs Pairs to format.
+     * @param dateFmt Formatter.
+     */
+    private void format(StringBuilder b, Collection<IgnitePair<Long>> pairs, SimpleDateFormat dateFmt) {
+        for (IgnitePair<Long> p : pairs) {
+            b.append(String.format("%15d", p.get1())).append(" ")
+                .append(dateFmt.format(new Date(p.get2()))).append(U.nl());
+        }
     }
 
     /** {@inheritDoc} */
@@ -678,8 +1093,22 @@ private void processRegularMessage(
                     msgC.run();
                 }
             }
+
+            @Override public String toString() {
+                return "Message closure [msg=" + msg + ']';
+            }
         };
 
+        if (msg.topicOrdinal() == TOPIC_IO_TEST.ordinal()) {
+            IgniteIoTestMessage msg0 = (IgniteIoTestMessage)msg.message();
+
+            if (msg0.processFromNioThread()) {
+                c.run();
+
+                return;
+            }
+        }
+
         try {
             pools.poolForPolicy(plc).execute(c);
         }
@@ -1796,8 +2225,7 @@ public boolean removeMessageListener(Object topic, @Nullable GridMessageListener
         if (rmv && log.isDebugEnabled())
             log.debug("Removed message listener [topic=" + topic + ", lsnr=" + lsnr + ']');
 
-        if (lsnr instanceof ArrayListener)
-        {
+        if (lsnr instanceof ArrayListener) {
             for (GridMessageListener childLsnr : ((ArrayListener)lsnr).arr)
                 closeListener(childLsnr);
         }
@@ -2460,4 +2888,269 @@ public UUID nodeId() {
             return S.toString(DelayedMessage.class, this, super.toString());
         }
     }
+
+    /**
+     *
+     */
+    private class IoTestFuture extends GridFutureAdapter<List<IgniteIoTestMessage>> {
+        /** */
+        private final long id;
+
+        /** */
+        private final int cntr;
+
+        /** */
+        private final List<IgniteIoTestMessage> ress;
+
+        /**
+         * @param id ID.
+         * @param cntr Counter.
+         */
+        IoTestFuture(long id, int cntr) {
+            assert cntr > 0 : cntr;
+
+            this.id = id;
+            this.cntr = cntr;
+
+            ress = new ArrayList<>(cntr);
+        }
+
+        /**
+         *
+         */
+        void onResponse(IgniteIoTestMessage res) {
+            boolean complete;
+
+            synchronized (this) {
+                ress.add(res);
+
+                complete = cntr == ress.size();
+            }
+
+            if (complete)
+                onDone(ress);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean onDone(List<IgniteIoTestMessage> res, @Nullable Throwable err) {
+            if (super.onDone(res, err)) {
+                ioTestMap().remove(id);
+
+                return true;
+            }
+
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(IoTestFuture.class, this);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class IoTestThreadLocalNodeResults {
+        /** */
+        private final long[] resLatency;
+
+        /** */
+        private final int rangesCnt;
+
+        /** */
+        private long totalLatency;
+
+        /** */
+        private long maxLatency;
+
+        /** */
+        private long maxLatencyTs;
+
+        /** */
+        private long maxReqSendQueueTime;
+
+        /** */
+        private long maxReqSendQueueTimeTs;
+
+        /** */
+        private long maxReqRcvQueueTime;
+
+        /** */
+        private long maxReqRcvQueueTimeTs;
+
+        /** */
+        private long maxResSendQueueTime;
+
+        /** */
+        private long maxResSendQueueTimeTs;
+
+        /** */
+        private long maxResRcvQueueTime;
+
+        /** */
+        private long maxResRcvQueueTimeTs;
+
+        /** */
+        private long maxReqWireTimeMillis;
+
+        /** */
+        private long maxReqWireTimeTs;
+
+        /** */
+        private long maxResWireTimeMillis;
+
+        /** */
+        private long maxResWireTimeTs;
+
+        /** */
+        private final long latencyLimit;
+
+        /**
+         * @param rangesCnt Ranges count.
+         * @param latencyLimit
+         */
+        public IoTestThreadLocalNodeResults(int rangesCnt, long latencyLimit) {
+            this.rangesCnt = rangesCnt;
+            this.latencyLimit = latencyLimit;
+
+            resLatency = new long[rangesCnt + 1];
+        }
+
+        /**
+         * @param msg
+         */
+        public void onResult(IgniteIoTestMessage msg) {
+            long now = System.currentTimeMillis();
+
+            long latency = msg.responseProcessedTs() - msg.requestCreateTs();
+
+            int idx = latency >= latencyLimit ?
+                rangesCnt /* Timed out. */ :
+                (int)Math.floor((1.0 * latency) / ((1.0 * latencyLimit) / rangesCnt));
+
+            resLatency[idx]++;
+
+            totalLatency += latency;
+
+            if (maxLatency < latency) {
+                maxLatency = latency;
+                maxLatencyTs = now;
+            }
+
+            long reqSndQueueTime = msg.requestSendTs() - msg.requestCreateTs();
+
+            if (maxReqSendQueueTime < reqSndQueueTime) {
+                maxReqSendQueueTime = reqSndQueueTime;
+                maxReqSendQueueTimeTs = now;
+            }
+
+            long reqRcvQueueTime = msg.requestProcessTs() - msg.requestReceiveTs();
+
+            if (maxReqRcvQueueTime < reqRcvQueueTime) {
+                maxReqRcvQueueTime = reqRcvQueueTime;
+                maxReqRcvQueueTimeTs = now;
+            }
+
+            long resSndQueueTime = msg.responseSendTs() - msg.requestProcessTs();
+
+            if (maxResSendQueueTime < resSndQueueTime) {
+                maxResSendQueueTime = resSndQueueTime;
+                maxResSendQueueTimeTs = now;
+            }
+
+            long resRcvQueueTime = msg.responseProcessedTs() - msg.responseReceiveTs();
+
+            if (maxResRcvQueueTime < resRcvQueueTime) {
+                maxResRcvQueueTime = resRcvQueueTime;
+                maxResRcvQueueTimeTs = now;
+            }
+
+            long reqWireTimeMillis = msg.requestReceivedTsMillis() - msg.requestSendTsMillis();
+
+            if (maxReqWireTimeMillis < reqWireTimeMillis) {
+                maxReqWireTimeMillis = reqWireTimeMillis;
+                maxReqWireTimeTs = now;
+            }
+
+            long resWireTimeMillis = msg.responseReceivedTsMillis() - msg.requestSendTsMillis();
+
+            if (maxResWireTimeMillis < resWireTimeMillis) {
+                maxResWireTimeMillis = resWireTimeMillis;
+                maxResWireTimeTs = now;
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private static class IoTestNodeResults {
+        /** */
+        private long latencyLimit;
+
+        /** */
+        private long[] resLatency;
+
+        /** */
+        private long totalLatency;
+
+        /** */
+        private Collection<IgnitePair<Long>> maxLatency = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxReqSendQueueTime = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxReqRcvQueueTime = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxResSendQueueTime = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxResRcvQueueTime = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxReqWireTimeMillis = new ArrayList<>();
+
+        /** */
+        private Collection<IgnitePair<Long>> maxResWireTimeMillis = new ArrayList<>();
+
+        /**
+         * @param res Node results to add.
+         */
+        public void add(IoTestThreadLocalNodeResults res) {
+            if (resLatency == null) {
+                resLatency = res.resLatency.clone();
+                latencyLimit = res.latencyLimit;
+            }
+            else {
+                assert latencyLimit == res.latencyLimit;
+                assert resLatency.length == res.resLatency.length;
+
+                for (int i = 0; i < resLatency.length; i++)
+                    resLatency[i] += res.resLatency[i];
+            }
+
+            totalLatency += res.totalLatency;
+
+            maxLatency.add(F.pair(res.maxLatency, res.maxLatencyTs));
+            maxReqSendQueueTime.add(F.pair(res.maxReqSendQueueTime, res.maxReqSendQueueTimeTs));
+            maxReqRcvQueueTime.add(F.pair(res.maxReqRcvQueueTime, res.maxReqRcvQueueTimeTs));
+            maxResSendQueueTime.add(F.pair(res.maxResSendQueueTime, res.maxResSendQueueTimeTs));
+            maxResRcvQueueTime.add(F.pair(res.maxResRcvQueueTime, res.maxResRcvQueueTimeTs));
+            maxReqWireTimeMillis.add(F.pair(res.maxReqWireTimeMillis, res.maxReqWireTimeTs));
+            maxResWireTimeMillis.add(F.pair(res.maxResWireTimeMillis, res.maxResWireTimeTs));
+        }
+
+        /**
+         * @return Bin latency in microseconds.
+         */
+        public long binLatencyMcs() {
+            if (resLatency == null)
+                throw new IllegalStateException();
+
+            return latencyLimit / (1000 * (resLatency.length - 1));
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index f36191cbafcd9..743d5f74d2ec9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -165,6 +165,11 @@ public GridIoMessageFactory(MessageFactory[] ext) {
         Message msg = null;
 
         switch (type) {
+            case -43:
+                msg = new IgniteIoTestMessage();
+
+                break;
+
             case -27:
                 msg = new GridDhtTxOnePhaseCommitAckRequest();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
new file mode 100644
index 0000000000000..6145439d87b4f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ *
+ */
+public class IgniteIoTestMessage implements Message {
+    /** */
+    private static byte FLAG_PROC_FROM_NIO = 1;
+
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private long id;
+
+    /** */
+    private byte flags;
+
+    /** */
+    private boolean req;
+
+    /** */
+    private byte payload[];
+
+    /** */
+    private long reqCreateTs;
+
+    /** */
+    private long reqSndTs;
+
+    /** */
+    private long reqSndTsMillis;
+
+    /** */
+    private long reqRcvTs;
+
+    /** */
+    private long reqRcvTsMillis;
+
+    /** */
+    private long reqProcTs;
+
+    /** */
+    private long resSndTs;
+
+    /** */
+    private long resSndTsMillis;
+
+    /** */
+    private long resRcvTs;
+
+    /** */
+    private long resRcvTsMillis;
+
+    /** */
+    private long resProcTs;
+
+    /** */
+    @GridDirectTransient
+    private UUID sndNodeId;
+
+    /**
+     *
+     */
+    public IgniteIoTestMessage() {
+        // No-op.
+    }
+
+    /**
+     * @param id Message ID.
+     * @param req Request flag.
+     * @param payload Payload.
+     */
+    public IgniteIoTestMessage(long id, boolean req, byte[] payload) {
+        this.id = id;
+        this.req = req;
+        this.payload = payload;
+
+        reqCreateTs = System.nanoTime();
+    }
+
+    /**
+     * @return {@code True} if message should be processed from NIO thread
+     * (otherwise message is submitted to system pool).
+     */
+    public boolean processFromNioThread() {
+        return isFlag(FLAG_PROC_FROM_NIO);
+    }
+
+    /**
+     * @param procFromNioThread {@code True} if message should be processed from NIO thread.
+     */
+    public void processFromNioThread(boolean procFromNioThread) {
+        setFlag(procFromNioThread, FLAG_PROC_FROM_NIO);
+    }
+
+    /**
+     * @param flags Flags.
+     */
+    public void flags(byte flags) {
+        this.flags = flags;
+    }
+
+    /**
+     * @return Flags.
+     */
+    public byte flags() {
+        return flags;
+    }
+
+    /**
+     * Sets flag mask.
+     *
+     * @param flag Set or clear.
+     * @param mask Mask.
+     */
+    private void setFlag(boolean flag, int mask) {
+        flags = flag ? (byte)(flags | mask) : (byte)(flags & ~mask);
+    }
+
+    /**
+     * Reads flag mask.
+     *
+     * @param mask Mask to read.
+     * @return Flag value.
+     */
+    private boolean isFlag(int mask) {
+        return (flags & mask) != 0;
+    }
+
+    /**
+     * @return {@code true} if this is request.
+     */
+    public boolean request() {
+        return req;
+    }
+
+    /**
+     * @return ID.
+     */
+    public long id() {
+        return id;
+    }
+
+    /**
+     * @return Request create timestamp.
+     */
+    public long requestCreateTs() {
+        return reqCreateTs;
+    }
+
+    /**
+     * @return Request send timestamp.
+     */
+    public long requestSendTs() {
+        return reqSndTs;
+    }
+
+    /**
+     * @return Request receive timestamp.
+     */
+    public long requestReceiveTs() {
+        return reqRcvTs;
+    }
+
+    /**
+     * @return Request process started timestamp.
+     */
+    public long requestProcessTs() {
+        return reqProcTs;
+    }
+
+    /**
+     * @return Response send timestamp.
+     */
+    public long responseSendTs() {
+        return resSndTs;
+    }
+
+    /**
+     * @return Response receive timestamp.
+     */
+    public long responseReceiveTs() {
+        return resRcvTs;
+    }
+
+    /**
+     * @return Response process timestamp.
+     */
+    public long responseProcessTs() {
+        return resProcTs;
+    }
+
+    /**
+     * @return Request send timestamp (millis).
+     */
+    public long requestSendTsMillis() {
+        return reqSndTsMillis;
+    }
+
+    /**
+     * @return Request received timestamp (millis).
+     */
+    public long requestReceivedTsMillis() {
+        return reqRcvTsMillis;
+    }
+
+    /**
+     * @return Response send timestamp (millis).
+     */
+    public long responseSendTsMillis() {
+        return resSndTsMillis;
+    }
+
+    /**
+     * @return Response received timestamp (millis).
+     */
+    public long responseReceivedTsMillis() {
+        return resRcvTsMillis;
+    }
+
+    /**
+     * This method is called to initialize tracing variables.
+     * TODO: introduce direct message lifecycle API?
+     */
+    public void onAfterRead() {
+        if (req && reqRcvTs == 0) {
+            reqRcvTs = System.nanoTime();
+
+            reqRcvTsMillis = System.currentTimeMillis();
+        }
+
+        if (!req && resRcvTs == 0) {
+            resRcvTs = System.nanoTime();
+
+            resRcvTsMillis = System.currentTimeMillis();
+        }
+    }
+
+    /**
+     * This method is called to initialize tracing variables.
+     * TODO: introduce direct message lifecycle API?
+     */
+    public void onBeforeWrite() {
+        if (req && reqSndTs == 0) {
+            reqSndTs = System.nanoTime();
+
+            reqSndTsMillis = System.currentTimeMillis();
+        }
+
+        if (!req && resSndTs == 0) {
+            resSndTs = System.nanoTime();
+
+            resSndTsMillis = System.currentTimeMillis();
+        }
+    }
+
+    /**
+     *
+     */
+    public void copyDataFromRequest(IgniteIoTestMessage req) {
+        reqCreateTs = req.reqCreateTs;
+
+        reqSndTs = req.reqSndTs;
+        reqSndTsMillis = req.reqSndTsMillis;
+
+        reqRcvTs = req.reqRcvTs;
+        reqRcvTsMillis = req.reqRcvTsMillis;
+    }
+
+    /**
+     *
+     */
+    public void onRequestProcessed() {
+        reqProcTs = System.nanoTime();
+    }
+
+    /**
+     *
+     */
+    public void onResponseProcessed() {
+        resProcTs = System.nanoTime();
+    }
+
+    /**
+     * @return Response processed timestamp.
+     */
+    public long responseProcessedTs() {
+        return resProcTs;
+    }
+
+    /**
+     * @return Sender node ID.
+     */
+    public UUID senderNodeId() {
+        return sndNodeId;
+    }
+
+    /**
+     * @param sndNodeId Sender node ID.
+     */
+    public void senderNodeId(UUID sndNodeId) {
+        this.sndNodeId = sndNodeId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        onBeforeWrite();
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeByte("flags", flags))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeLong("id", id))
+                    return false;
+
+                writer.incrementState();
+
+            case 2:
+                if (!writer.writeByteArray("payload", payload))
+                    return false;
+
+                writer.incrementState();
+
+            case 3:
+                if (!writer.writeBoolean("req", req))
+                    return false;
+
+                writer.incrementState();
+
+            case 4:
+                if (!writer.writeLong("reqCreateTs", reqCreateTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 5:
+                if (!writer.writeLong("reqProcTs", reqProcTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 6:
+                if (!writer.writeLong("reqRcvTs", reqRcvTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 7:
+                if (!writer.writeLong("reqRcvTsMillis", reqRcvTsMillis))
+                    return false;
+
+                writer.incrementState();
+
+            case 8:
+                if (!writer.writeLong("reqSndTs", reqSndTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 9:
+                if (!writer.writeLong("reqSndTsMillis", reqSndTsMillis))
+                    return false;
+
+                writer.incrementState();
+
+            case 10:
+                if (!writer.writeLong("resProcTs", resProcTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 11:
+                if (!writer.writeLong("resRcvTs", resRcvTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 12:
+                if (!writer.writeLong("resRcvTsMillis", resRcvTsMillis))
+                    return false;
+
+                writer.incrementState();
+
+            case 13:
+                if (!writer.writeLong("resSndTs", resSndTs))
+                    return false;
+
+                writer.incrementState();
+
+            case 14:
+                if (!writer.writeLong("resSndTsMillis", resSndTsMillis))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                flags = reader.readByte("flags");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                id = reader.readLong("id");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 2:
+                payload = reader.readByteArray("payload");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 3:
+                req = reader.readBoolean("req");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 4:
+                reqCreateTs = reader.readLong("reqCreateTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 5:
+                reqProcTs = reader.readLong("reqProcTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 6:
+                reqRcvTs = reader.readLong("reqRcvTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 7:
+                reqRcvTsMillis = reader.readLong("reqRcvTsMillis");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 8:
+                reqSndTs = reader.readLong("reqSndTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 9:
+                reqSndTsMillis = reader.readLong("reqSndTsMillis");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 10:
+                resProcTs = reader.readLong("resProcTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 11:
+                resRcvTs = reader.readLong("resRcvTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 12:
+                resRcvTsMillis = reader.readLong("resRcvTsMillis");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 13:
+                resSndTs = reader.readLong("resSndTs");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 14:
+                resSndTsMillis = reader.readLong("resSndTsMillis");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        onAfterRead();
+
+        return reader.afterMessageRead(IgniteIoTestMessage.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte directType() {
+        return -43;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 15;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IgniteIoTestMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
index 0754c27ee05bd..0a73d5ed61a0f 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
@@ -372,4 +372,48 @@ public interface IgniteMXBean {
      */
     @MXBeanDescription("Dumps debug information for the current node.")
     public void dumpDebugInfo();
+
+    /**
+     * Runs IO latency test against all remote server nodes in cluster.
+     *
+     * @param warmup Warmup duration in milliseconds.
+     * @param duration Test duration in milliseconds.
+     * @param threads Thread count.
+     * @param maxLatency Max latency in nanoseconds.
+     * @param rangesCnt Ranges count in resulting histogram.
+     * @param payLoadSize Payload size in bytes.
+     * @param procFromNioThread {@code True} to process requests in NIO threads.
+     */
+    @MXBeanDescription("Runs IO latency test against all remote server nodes in cluster.")
+    @MXBeanParametersNames(
+        {
+            "warmup",
+            "duration",
+            "threads",
+            "maxLatency",
+            "rangesCnt",
+            "payLoadSize",
+            "procFromNioThread"
+        }
+    )
+    @MXBeanParametersDescriptions(
+        {
+            "Warmup duration (millis).",
+            "Test duration (millis).",
+            "Threads count.",
+            "Maximum latency expected (nanos).",
+            "Ranges count for histogram.",
+            "Payload size (bytes).",
+            "Process requests in NIO-threads flag."
+        }
+    )
+    void runIoTest(
+        long warmup,
+        long duration,
+        int threads,
+        long maxLatency,
+        int rangesCnt,
+        int payLoadSize,
+        boolean procFromNioThread
+    );
 }

From 540ca449f1bd2386d3ba0722afb21dd3a504d044 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 13 Jun 2017 20:55:38 +0300
Subject: [PATCH 224/516] GG-12355: Added discovery ring latency test + made it
 available from MBean (cherry-picked from master).

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 20 +++++
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 57 +++++++++++++-
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |  6 ++
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |  5 ++
 .../discovery/tcp/TcpDiscoverySpiMBean.java   | 22 ++++++
 .../TcpDiscoveryRingLatencyCheckMessage.java  | 77 +++++++++++++++++++
 6 files changed, 186 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryRingLatencyCheckMessage.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index b075dc1fb5289..71550447fa04d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -97,6 +97,7 @@
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeLeftMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingRequest;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingResponse;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryRingLatencyCheckMessage;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -834,6 +835,16 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
             U.closeQuiet(sockStream.socket());
     }
 
+    /** {@inheritDoc} */
+    @Override public void checkRingLatency(int maxHops) {
+        TcpDiscoveryRingLatencyCheckMessage msg = new TcpDiscoveryRingLatencyCheckMessage(getLocalNodeId(), maxHops);
+
+        if (log.isInfoEnabled())
+            log.info("Latency check initiated: " + msg.id());
+
+        sockWriter.sendMessage(msg);
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteSpiThread workerThread() {
         return msgWorker;
@@ -1228,6 +1239,12 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
                         msg,
                         sockTimeout);
 
+                    IgniteUuid latencyCheckId = msg instanceof TcpDiscoveryRingLatencyCheckMessage ?
+                        msg.id() : null;
+
+                    if (latencyCheckId != null && log.isInfoEnabled())
+                        log.info("Latency check message has been written to socket: " + latencyCheckId);
+
                     msg = null;
 
                     if (ack) {
@@ -1255,6 +1272,9 @@ void ackReceived(TcpDiscoveryClientAckResponse res) {
 
                             throw new IOException("Failed to get acknowledge for message: " + unacked);
                         }
+
+                        if (latencyCheckId != null && log.isInfoEnabled())
+                            log.info("Latency check message has been acked: " + latencyCheckId);
                     }
                 }
                 catch (InterruptedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 58b362ffd0f15..6bdd80cbe6ff6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -129,6 +129,7 @@
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingRequest;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryPingResponse;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryRedirectToClient;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryRingLatencyCheckMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryStatusCheckMessage;
 import org.apache.ignite.thread.IgniteThreadPoolExecutor;
 import org.jetbrains.annotations.Nullable;
@@ -1583,6 +1584,16 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void checkRingLatency(int maxHops) {
+        TcpDiscoveryRingLatencyCheckMessage msg = new TcpDiscoveryRingLatencyCheckMessage(getLocalNodeId(), maxHops);
+
+        if (log.isInfoEnabled())
+            log.info("Latency check initiated: " + msg.id());
+
+        msgWorker.addMessage(msg);
+    }
+
     /** {@inheritDoc} */
     @Override void simulateNodeFailure() {
         U.warn(log, "Simulating node failure: " + getLocalNodeId());
@@ -2647,6 +2658,9 @@ else if (msg instanceof TcpDiscoveryCustomEventMessage)
             else if (msg instanceof TcpDiscoveryClientPingRequest)
                 processClientPingRequest((TcpDiscoveryClientPingRequest)msg);
 
+            else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage)
+                processRingLatencyCheckMessage((TcpDiscoveryRingLatencyCheckMessage)msg);
+
             else
                 assert false : "Unknown message type: " + msg.getClass().getSimpleName();
 
@@ -3054,12 +3068,20 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof
                                     }
                                 }
 
+                                boolean latencyCheck = msg instanceof TcpDiscoveryRingLatencyCheckMessage;
+
+                                if (latencyCheck && log.isInfoEnabled())
+                                    log.info("Latency check message has been written to socket: " + msg.id());
+
                                 spi.writeToSocket(sock, out, msg, timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()));
 
                                 long tstamp0 = U.currentTimeMillis();
 
                                 int res = spi.readReceipt(sock, timeoutHelper.nextTimeoutChunk(ackTimeout0));
 
+                                if (latencyCheck && log.isInfoEnabled())
+                                    log.info("Latency check message has been acked: " + msg.id());
+
                                 spi.stats.onMessageSent(msg, tstamp0 - tstamp, U.currentTimeMillis() - tstamp0);
 
                                 onMessageExchanged();
@@ -4477,6 +4499,33 @@ private void processNodeAddFinishedMessage(TcpDiscoveryNodeAddFinishedMessage ms
             checkPendingCustomMessages();
         }
 
+        /**
+         * Processes latency check message.
+         *
+         * @param msg Latency check message.
+         */
+        private void processRingLatencyCheckMessage(TcpDiscoveryRingLatencyCheckMessage msg) {
+            assert msg != null;
+
+            if (msg.maxHopsReached()) {
+                if (log.isInfoEnabled())
+                    log.info("Latency check has been discarded (max hops reached) [id=" + msg.id() +
+                        ", maxHops=" + msg.maxHops() + ']');
+
+                return;
+            }
+
+            if (log.isInfoEnabled())
+                log.info("Latency check processing: " + msg.id());
+
+            if (sendMessageToRemotes(msg))
+                sendMessageAcrossRing(msg);
+            else {
+                if (log.isInfoEnabled())
+                    log.info("Latency check has been discarded (no remote nodes): " + msg.id());
+            }
+        }
+
         /**
          * Processes node left message.
          *
@@ -6056,7 +6105,7 @@ else if (msg instanceof TcpDiscoveryLoopbackProblemMessage) {
 
                             continue;
                         }
-                        if (msg instanceof TcpDiscoveryPingResponse) {
+                        else if (msg instanceof TcpDiscoveryPingResponse) {
                             assert msg.client() : msg;
 
                             ClientMessageWorker clientWorker = clientMsgWorkers.get(msg.creatorNodeId());
@@ -6066,6 +6115,12 @@ else if (msg instanceof TcpDiscoveryLoopbackProblemMessage) {
 
                             continue;
                         }
+                        else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage) {
+                            if (log.isInfoEnabled())
+                                log.info("Latency check message has been read: " + msg.id());
+
+                            ((TcpDiscoveryRingLatencyCheckMessage)msg).onRead();
+                        }
 
                         TcpDiscoveryClientHeartbeatMessage heartbeatMsg = null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index 84c2ff28afef0..cb85dc1c44b86 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryRingLatencyCheckMessage;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -281,6 +282,11 @@ protected static String threadStatus(Thread t) {
      */
     public abstract void brakeConnection();
 
+    /**
+     * @param maxHops Maximum hops for {@link TcpDiscoveryRingLatencyCheckMessage}.
+     */
+    public abstract void checkRingLatency(int maxHops);
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index a2a47feb31486..cefbcd675a303 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1160,6 +1160,11 @@ LinkedHashSet<InetSocketAddress> getNodeAddresses(TcpDiscoveryNode node, boolean
         return stats.coordinatorSinceTimestamp();
     }
 
+    /** {@inheritDoc} */
+    @Override public void checkRingLatency(int maxHops) {
+        impl.checkRingLatency(maxHops);
+    }
+
     /** {@inheritDoc} */
     @Override protected void onContextInitialized0(IgniteSpiContext spiCtx) throws IgniteSpiException {
         super.onContextInitialized0(spiCtx);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
index 1427929a4cb88..e96088dd99d37 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
@@ -20,6 +20,8 @@
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.mxbean.MXBeanDescription;
+import org.apache.ignite.mxbean.MXBeanParametersDescriptions;
+import org.apache.ignite.mxbean.MXBeanParametersNames;
 import org.apache.ignite.spi.IgniteSpiManagementMBean;
 import org.jetbrains.annotations.Nullable;
 
@@ -281,4 +283,24 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
      */
     @MXBeanDescription("Client mode.")
     public boolean isClientMode() throws IllegalStateException;
+
+    /**
+     * Diagnosis method for determining ring message latency.
+     * On this method call special message will be sent across the ring
+     * and stats about the message will appear in the logs of each node.
+     *
+     * @param maxHops Maximum hops for the message (3 * TOTAL_NODE_CNT is recommended).
+     */
+    @MXBeanDescription("Check ring latency.")
+    @MXBeanParametersNames(
+        {
+            "maxHops"
+        }
+    )
+    @MXBeanParametersDescriptions(
+        {
+            "Maximum hops for the message (3 * TOTAL_NODE_CNT is recommended)."
+        }
+    )
+    public void checkRingLatency(int maxHops);
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryRingLatencyCheckMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryRingLatencyCheckMessage.java
new file mode 100644
index 0000000000000..d8c11452e641e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryRingLatencyCheckMessage.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.messages;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class TcpDiscoveryRingLatencyCheckMessage extends TcpDiscoveryAbstractMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private int maxHops;
+
+    /** */
+    private int curHop;
+
+    /**
+     * @param creatorNodeId Creator node ID.
+     * @param maxHops Max hops for this message.
+     */
+    public TcpDiscoveryRingLatencyCheckMessage(
+        UUID creatorNodeId,
+        int maxHops
+    ) {
+        super(creatorNodeId);
+
+        assert maxHops > 0;
+
+        this.maxHops = maxHops;
+    }
+
+    /**
+     *
+     */
+    public void onRead() {
+        curHop++;
+    }
+
+    /**
+     * @return Max hops.
+     */
+    public int maxHops() {
+        return maxHops;
+    }
+
+    /**
+     * @return {@code True} if max hops has been reached.
+     */
+    public boolean maxHopsReached() {
+        return curHop == maxHops;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(TcpDiscoveryRingLatencyCheckMessage.class, this, "super", super.toString());
+    }
+}

From 8cd9e829380f4c91cc9bb126169863286d1cb323 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Jun 2017 15:40:14 +0300
Subject: [PATCH 225/516] GG-12353: Added local binary context flag.

Backport of IGNITE-5223 with fixes.
---
 .../apache/ignite/IgniteSystemProperties.java |  10 +-
 .../internal/binary/BinaryReaderExImpl.java   |   2 +-
 .../BinaryMarshallerLocalMetadataCache.java   | 126 ++++++++++++++++++
 3 files changed, 128 insertions(+), 10 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index f35e9e620070a..775aae03a1ef4 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -320,14 +320,6 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_H2_DEBUG_CONSOLE = "IGNITE_H2_DEBUG_CONSOLE";
 
-    /**
-     * This property allows to specify user defined port which H2 indexing SPI will use
-     * to start H2 debug console on. If this property is not set or set to 0, H2 debug
-     * console will use system-provided dynamic port.
-     * This property is only relevant when {@link #IGNITE_H2_DEBUG_CONSOLE} property is set.
-     */
-    public static final String IGNITE_H2_DEBUG_CONSOLE_PORT = "IGNITE_H2_DEBUG_CONSOLE_PORT";
-
     /**
      * If this property is set to {@code true} then shared memory space native debug will be enabled.
      */
@@ -511,7 +503,7 @@ public final class IgniteSystemProperties {
     /**
      * Whether Ignite can access unaligned memory addresses.
      * <p>
-     * Defaults to {@code false}, meaning that unaligned access will be performed only on x86 architecture.
+     * Defaults to {@code} false, meaning that unaligned access will be performed only on x86 architecture.
      */
     public static final String IGNITE_UNALIGNED_MEMORY_ACCESS = "IGNITE_UNALIGNED_MEMORY_ACCESS";
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
index ad2e736fa5f34..7f71b460e5223 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
@@ -1723,7 +1723,7 @@ public BinarySchema getOrCreateSchema() {
             if (fieldIdLen != BinaryUtils.FIELD_ID_LEN) {
                 BinaryTypeImpl type = (BinaryTypeImpl)ctx.metadata(typeId);
 
-                if (type == null || type.metadata() == null || type.metadata().schemas().isEmpty()) {
+                if (type == null || type.metadata() == null || type.metadata().schemas() == null || type.metadata().schemas().isEmpty()) {
                     if (IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE, false)) {
                         BinaryClassDescriptor desc = ctx.descriptorForTypeId(true, typeId, getClass().getClassLoader(), false);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java
new file mode 100644
index 0000000000000..8a2d38b61a48b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.testframework.junits.IgniteTestResources;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE property.
+ */
+public class BinaryMarshallerLocalMetadataCache extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName,
+        IgniteTestResources rsrcs) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName, rsrcs);
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE, "true");
+
+        startGrid(0);
+        startGrid(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        try {
+            stopAllGrids();
+        }
+        finally {
+            System.clearProperty(IgniteSystemProperties.IGNITE_USE_LOCAL_BINARY_MARSHALLER_CACHE);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLocalMetadata() throws Exception {
+        final BinaryObject obj = grid(0).binary().toBinary(new OptimizedContainer(new Optimized()));
+
+        ClusterGroup remotes = grid(0).cluster().forRemotes();
+
+        OptimizedContainer res = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
+            @Override public OptimizedContainer call() throws Exception {
+
+                return obj.deserialize();
+            }
+        });
+
+        OptimizedContainer res2 = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
+            @Override public OptimizedContainer call() throws Exception {
+
+                return obj.deserialize();
+            }
+        });
+
+        System.out.println(res);
+        System.out.println(res2);
+    }
+
+    /**
+     *
+     */
+    private static class OptimizedContainer {
+        /** */
+        private Optimized optim;
+
+        /**
+         * @param optim Val.
+         */
+        public OptimizedContainer(Optimized optim) {
+            this.optim = optim;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Optimized implements Externalizable {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private String fld;
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(ObjectOutput out) throws IOException {
+            U.writeUTFStringNullable(out, fld);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+            fld = U.readUTFStringNullable(in);
+        }
+    }
+}

From f8224d13cf9a6432ba65e0016370ba51bbb544e9 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 15 Jun 2017 22:49:45 +0300
Subject: [PATCH 226/516] GG-12299: Make sure concurrent type registrations do
 not trigger multiple cache updates.

---
 .../internal/MarshallerContextAdapter.java    | 134 ++++++++++++++----
 .../BinaryMarshallerLocalMetadataCache.java   |  63 +++++---
 2 files changed, 149 insertions(+), 48 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java
index ad3439325d446..56007adb34544 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java
@@ -32,6 +32,8 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap2;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.IgniteOutClosureX;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.MarshallerContext;
 import org.apache.ignite.plugin.PluginProvider;
@@ -49,7 +51,7 @@ public abstract class MarshallerContextAdapter implements MarshallerContext {
     private static final String JDK_CLS_NAMES_FILE = "META-INF/classnames-jdk.properties";
 
     /** */
-    private final ConcurrentMap<Integer, String> map = new ConcurrentHashMap8<>();
+    private final ConcurrentMap<Integer, Object> map = new ConcurrentHashMap8<>();
 
     /** */
     private final Set<String> registeredSystemTypes = new HashSet<>();
@@ -134,55 +136,129 @@ private void processResource(URL url) throws IOException {
 
                 int typeId = clsName.hashCode();
 
-                String oldClsName;
+                Object oldClsNameOrFuture = map.put(typeId, clsName);
 
-                if ((oldClsName = map.put(typeId, clsName)) != null) {
-                    if (!oldClsName.equals(clsName))
-                        throw new IgniteException("Duplicate type ID [id=" + typeId + ", clsName=" + clsName +
-                        ", oldClsName=" + oldClsName + ']');
-                }
+                try {
+                    String oldClsName = unwrap(oldClsNameOrFuture);
+
+                    if (oldClsName != null) {
+                        if (!oldClsName.equals(clsName))
+                            throw new IgniteException("Duplicate type ID [id=" + typeId + ", clsName=" + clsName +
+                            ", oldClsName=" + oldClsName + ']');
+                    }
 
-                registeredSystemTypes.add(clsName);
+                    registeredSystemTypes.add(clsName);
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IllegalStateException("Failed to process type ID [typeId=" + typeId +
+                        ", clsName" + clsName + ']', e);
+                }
             }
         }
     }
 
     /** {@inheritDoc} */
-    @Override public boolean registerClass(int id, Class cls) throws IgniteCheckedException {
-        boolean registered = true;
-
-        String clsName = map.get(id);
+    @Override public boolean registerClass(final int id, final Class cls) throws IgniteCheckedException {
+        Object clsNameOrFuture = map.get(id);
+
+        String clsName = clsNameOrFuture != null ?
+            unwrap(clsNameOrFuture) :
+            computeIfAbsent(id, new IgniteOutClosureX<String>() {
+                @Override public String applyx() throws IgniteCheckedException {
+                    return registerClassName(id, cls.getName()) ? cls.getName() : null;
+                }
+            });
 
-        if (clsName == null) {
-            registered = registerClassName(id, cls.getName());
+        // The only way we can have clsName eq null here is a failing concurrent thread.
+        if (clsName == null)
+            return false;
 
-            if (registered)
-                map.putIfAbsent(id, cls.getName());
-        }
-        else if (!clsName.equals(cls.getName()))
+        if (!clsName.equals(cls.getName()))
             throw new IgniteCheckedException("Duplicate ID [id=" + id + ", oldCls=" + clsName +
                 ", newCls=" + cls.getName());
 
-        return registered;
+        return true;
     }
 
     /** {@inheritDoc} */
-    @Override public Class getClass(int id, ClassLoader ldr) throws ClassNotFoundException, IgniteCheckedException {
-        String clsName = map.get(id);
+    @Override public Class getClass(final int id, ClassLoader ldr) throws ClassNotFoundException, IgniteCheckedException {
+        Object clsNameOrFuture = map.get(id);
+
+        String clsName = clsNameOrFuture != null ?
+            unwrap(clsNameOrFuture) :
+            computeIfAbsent(id, new IgniteOutClosureX<String>() {
+                @Override public String applyx() throws IgniteCheckedException {
+                    return className(id);
+                }
+            });
 
-        if (clsName == null) {
-            clsName = className(id);
+        if (clsName == null)
+            throw new ClassNotFoundException("Unknown type ID: " + id);
 
-            if (clsName == null)
-                throw new ClassNotFoundException("Unknown type ID: " + id);
+        return U.forName(clsName, ldr);
+    }
+
+    /**
+     * Computes the map value for the given ID. Will make sure that if there are two threads are calling
+     *      {@code computeIfAbsent}, only one of them will invoke the closure. If the closure threw an exeption,
+     *      all threads attempting to compute the value will throw this exception.
+     *
+     * @param id Type ID.
+     * @param clo Close to compute.
+     * @return Computed value.
+     * @throws IgniteCheckedException If closure threw an exception.
+     */
+    private String computeIfAbsent(int id, IgniteOutClosureX<String> clo) throws IgniteCheckedException {
+        Object clsNameOrFuture = map.get(id);
 
-            String old = map.putIfAbsent(id, clsName);
+        if (clsNameOrFuture == null) {
+            GridFutureAdapter<String> fut = new GridFutureAdapter<>();
 
-            if (old != null)
-                clsName = old;
+            Object old = map.putIfAbsent(id, fut);
+
+            if (old == null) {
+                String clsName = null;
+
+                try {
+                    try {
+                        clsName = clo.applyx();
+
+                        fut.onDone(clsName);
+
+                        clsNameOrFuture = clsName;
+                    }
+                    catch (Throwable e) {
+                        fut.onDone(e);
+
+                        throw e;
+                    }
+                }
+                finally {
+                    if (clsName != null)
+                        map.replace(id, fut, clsName);
+                    else
+                        map.remove(id, fut);
+                }
+            }
+            else
+                clsNameOrFuture = old;
         }
 
-        return U.forName(clsName, ldr);
+        // Unwrap the existing object.
+        return unwrap(clsNameOrFuture);
+    }
+
+    /**
+     * Unwraps an object into the String. Expects the object be {@code null}, a String or a GridFutureAdapter.
+     *
+     * @param clsNameOrFuture Class name or future to unwrap.
+     * @return Unwrapped value.
+     * @throws IgniteCheckedException If future completed with an exception.
+     */
+    private String unwrap(Object clsNameOrFuture) throws IgniteCheckedException {
+        return clsNameOrFuture == null ? null :
+            clsNameOrFuture instanceof String ? (String)clsNameOrFuture :
+                ((GridFutureAdapter<String>)clsNameOrFuture).get();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java
index 8a2d38b61a48b..571091e004567 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerLocalMetadataCache.java
@@ -21,12 +21,19 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.util.Collections;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.binary.BinaryObject;
-import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.MarshallerContextAdapter;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.marshaller.MarshallerContext;
+import org.apache.ignite.plugin.PluginProvider;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.IgniteTestResources;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -39,7 +46,9 @@ public class BinaryMarshallerLocalMetadataCache extends GridCommonAbstractTest {
         IgniteTestResources rsrcs) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName, rsrcs);
 
-        cfg.setMarshaller(new BinaryMarshaller());
+        cfg.setMarshaller(new BinaryMarshallerWrapper());
+
+        cfg.setCacheConfiguration(new CacheConfiguration().setName("part").setBackups(1));
 
         return cfg;
     }
@@ -66,26 +75,20 @@ public class BinaryMarshallerLocalMetadataCache extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testLocalMetadata() throws Exception {
-        final BinaryObject obj = grid(0).binary().toBinary(new OptimizedContainer(new Optimized()));
-
-        ClusterGroup remotes = grid(0).cluster().forRemotes();
+        final CyclicBarrier bar = new CyclicBarrier(64);
 
-        OptimizedContainer res = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
-            @Override public OptimizedContainer call() throws Exception {
-
-                return obj.deserialize();
-            }
-        });
+        awaitPartitionMapExchange();
 
-        OptimizedContainer res2 = grid(0).compute(remotes).call(new IgniteCallable<OptimizedContainer>() {
-            @Override public OptimizedContainer call() throws Exception {
+        GridTestUtils.runMultiThreaded(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                bar.await();
 
-                return obj.deserialize();
+                return grid(0).binary().toBinary(new OptimizedContainer(new Optimized()));
             }
-        });
+        }, 64, "async-runner");
 
-        System.out.println(res);
-        System.out.println(res2);
+        // We expect 3 here because Externalizable classes are registered twice with different type IDs.
+        assertEquals(3, ((BinaryMarshallerWrapper)grid(0).configuration().getMarshaller()).registerClassCalled.get());
     }
 
     /**
@@ -123,4 +126,26 @@ private static class Optimized implements Externalizable {
             fld = U.readUTFStringNullable(in);
         }
     }
+
+    private static class BinaryMarshallerWrapper extends BinaryMarshaller {
+        private MarshallerContext ctx0 = new MarshallerContextAdapter(Collections.<PluginProvider>emptyList()) {
+            @Override protected boolean registerClassName(int id, String clsName) {
+                U.dumpStack(id + " " + clsName);
+                registerClassCalled.incrementAndGet();
+
+                return true;
+            }
+
+            @Override protected String className(int id) throws IgniteCheckedException {
+                return null;
+            }
+        };
+
+        private AtomicInteger registerClassCalled = new AtomicInteger();
+
+        /** {@inheritDoc} */
+        @Override public MarshallerContext getContext() {
+            return ctx0;
+        }
+    }
 }

From 4ffc3acfa1bc43bea8c79bfd1864787c15cfc4de Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 20 Jun 2017 07:59:09 +0300
Subject: [PATCH 227/516] IGNITE-5528 - IS_EVICT_DISABLED flag is not cleared
 on cache store exception.

---
 .../processors/cache/GridCacheAdapter.java    |  62 ++++---
 .../processors/cache/GridCacheEntryEx.java    |   3 +-
 .../processors/cache/GridCacheMapEntry.java   |   2 +-
 ...tionEvictionDuringReadThroughSelfTest.java | 160 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |   3 +
 5 files changed, 205 insertions(+), 25 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 47c9998239777..220c192f21c46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2061,35 +2061,18 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                     }
                                 });
 
-                                if (loaded.size() != loadKeys.size()) {
-                                    boolean needTouch =
-                                        tx0 == null || (!tx0.implicit() && tx0.isolation() == READ_COMMITTED);
-
-                                    for (Map.Entry<KeyCacheObject, EntryGetResult> e : loadKeys.entrySet()) {
-                                        if (loaded.contains(e.getKey()))
-                                            continue;
-
-                                        if (needTouch || e.getValue().reserved()) {
-                                            GridCacheEntryEx entry = peekEx(e.getKey());
-
-                                            if (entry != null) {
-                                                if (e.getValue().reserved())
-                                                    entry.clearReserveForLoad(e.getValue().version());
-
-                                                if (needTouch)
-                                                    ctx.evicts().touch(entry, topVer);
-                                            }
-                                        }
-                                    }
-                                }
+                                clearReservationsIfNeeded(topVer, loadKeys, loaded, tx0);
 
                                 return map;
                             }
                         }), true),
                         new C2<Map<K, V>, Exception, IgniteInternalFuture<Map<K, V>>>() {
                             @Override public IgniteInternalFuture<Map<K, V>> apply(Map<K, V> map, Exception e) {
-                                if (e != null)
+                                if (e != null) {
+                                    clearReservationsIfNeeded(topVer, loadKeys, loaded, tx0);
+
                                     return new GridFinishedFuture<>(e);
+                                }
 
                                 if (tx0 == null || (!tx0.implicit() && tx0.isolation() == READ_COMMITTED)) {
                                     Collection<KeyCacheObject> notFound = new HashSet<>(loadKeys.keySet());
@@ -2146,6 +2129,41 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
         }
     }
 
+    /**
+     * @param topVer Affinity topology version for which load was performed.
+     * @param loadKeys Keys to load.
+     * @param loaded Actually loaded keys.
+     * @param tx0 Transaction within which the load was run, if any.
+     */
+    private void clearReservationsIfNeeded(
+        AffinityTopologyVersion topVer,
+        Map<KeyCacheObject, EntryGetResult> loadKeys,
+        Collection<KeyCacheObject> loaded,
+        IgniteTxLocalAdapter tx0
+    ) {
+        if (loaded.size() != loadKeys.size()) {
+            boolean needTouch =
+                tx0 == null || (!tx0.implicit() && tx0.isolation() == READ_COMMITTED);
+
+            for (Map.Entry<KeyCacheObject, EntryGetResult> e : loadKeys.entrySet()) {
+                if (loaded.contains(e.getKey()))
+                    continue;
+
+                if (needTouch || e.getValue().reserved()) {
+                    GridCacheEntryEx entry = peekEx(e.getKey());
+
+                    if (entry != null) {
+                        if (e.getValue().reserved())
+                            entry.clearReserveForLoad(e.getValue().version());
+
+                        if (needTouch)
+                            ctx.evicts().touch(entry, topVer);
+                    }
+                }
+            }
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public final V getAndPut(K key, V val) throws IgniteCheckedException {
         return getAndPut(key, val, null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index ccd22850d9eb7..2574ee2623d0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -363,9 +363,8 @@ public EntryGetResult innerGetAndReserveForLoad(boolean readSwap,
 
     /**
      * @param ver Expected entry version.
-     * @throws IgniteCheckedException If failed.
      */
-    public void clearReserveForLoad(GridCacheVersion ver) throws IgniteCheckedException;
+    public void clearReserveForLoad(GridCacheVersion ver);
 
     /**
      * Reloads entry from underlying storage.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index ea01bca4bd50b..1b2d63bc624f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -3621,7 +3621,7 @@ private long nextPartCounter(AffinityTopologyVersion topVer) {
     }
 
     /** {@inheritDoc} */
-    @Override public synchronized void clearReserveForLoad(GridCacheVersion ver) throws IgniteCheckedException {
+    @Override public synchronized void clearReserveForLoad(GridCacheVersion ver) {
         if (obsoleteVersionExtras() != null)
             return;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java
new file mode 100644
index 0000000000000..d5351f7e37776
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.LinkedHashSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.expiry.CreatedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class GridCachePartitionEvictionDuringReadThroughSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration<Integer, Integer> ccfg =
+            new CacheConfiguration<Integer, Integer>()
+                .setName("config")
+                .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+                .setBackups(0) // No need for backup, just load from the store if needed
+                .setCacheStoreFactory(new CacheStoreFactory())
+                .setEvictionPolicy(new LruEvictionPolicy(100))
+                .setNearConfiguration(new NearCacheConfiguration<Integer, Integer>()
+                .setNearEvictionPolicy(new LruEvictionPolicy<Integer, Integer>()));
+
+        ccfg.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.MINUTES, 1)))
+            .setReadThrough(true)
+            .setWriteThrough(false);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testPartitionRent() throws Exception {
+        startGrid(0);
+
+        final AtomicBoolean done = new AtomicBoolean();
+
+        IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Integer>() {
+            @Override
+            public Integer call() throws Exception {
+                LinkedHashSet<Integer> set = new LinkedHashSet<>();
+
+                set.add(1);
+                set.add(2);
+                set.add(3);
+                set.add(4);
+                set.add(5);
+
+                while (!done.get()) {
+                    try {
+                        grid(0).<Integer, Integer>cache("config").getAll(set);
+                    }
+                    catch (Throwable ignore) {
+                        // No-op.
+                    }
+                }
+
+                return null;
+            }
+        }, 4, "loader");
+
+        IgniteInternalFuture<Void> startFut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                for (int i = 1; i < 5; i++) {
+                    startGrid(i);
+
+                    awaitPartitionMapExchange();
+                }
+
+                return null;
+            }
+        });
+
+        startFut.get();
+
+        done.set(true);
+
+        fut.get();
+    }
+
+    /**
+     *
+     */
+    private static class CacheStoreFactory implements Factory<CacheStore<Integer, Integer>> {
+        /** {@inheritDoc} */
+        @Override public CacheStore<Integer, Integer> create() {
+            return new HangingCacheStore();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class HangingCacheStore extends CacheStoreAdapter<Integer, Integer> {
+        /** */
+        private CountDownLatch releaseLatch = new CountDownLatch(1);
+
+        /** {@inheritDoc} */
+        @Override public Integer load(Integer key) throws CacheLoaderException {
+            if (key == 3)
+                throw new CacheLoaderException();
+
+            return key;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Integer, ? extends Integer> entry) throws CacheWriterException {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 7582f5cc3ee88..ab0007b2f1459 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentFairAffinityTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest;
+import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest;
 import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest;
@@ -59,6 +60,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(IgniteCacheReadThroughEvictionsVariationsSuite.suite());
         suite.addTestSuite(IgniteCacheTxIteratorSelfTest.class);
 
+        suite.addTestSuite(GridCachePartitionEvictionDuringReadThroughSelfTest.class);
+
         return suite;
     }
 }

From 9036ad239d68eff663bc73a81baab2826b054d9a Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Jun 2017 18:25:31 +0300
Subject: [PATCH 228/516] Added MBean for system cache executors.

---
 .../apache/ignite/internal/IgniteKernal.java  | 38 +++++++++++++++----
 1 file changed, 30 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 4237aeebe49ce..2aa05f1b9ab11 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -61,10 +61,10 @@
 import org.apache.ignite.IgniteEvents;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteFileSystem;
+import org.apache.ignite.IgniteLock;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.IgniteQueue;
-import org.apache.ignite.IgniteLock;
 import org.apache.ignite.IgniteScheduler;
 import org.apache.ignite.IgniteSemaphore;
 import org.apache.ignite.IgniteServices;
@@ -115,7 +115,6 @@
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
 import org.apache.ignite.internal.processors.hadoop.Hadoop;
-import org.apache.ignite.internal.processors.hadoop.HadoopClassLoader;
 import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.job.GridJobProcessor;
 import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor;
@@ -175,6 +174,7 @@
 import org.apache.ignite.thread.IgniteStripedThreadPoolExecutor;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONFIG_URL;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DAEMON;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_NO_ASCII;
@@ -183,7 +183,6 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_STARVATION_CHECK_INTERVAL;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SUCCESS_FILE;
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.IgniteSystemProperties.getBoolean;
 import static org.apache.ignite.IgniteSystemProperties.snapshot;
 import static org.apache.ignite.internal.GridKernalState.DISCONNECTED;
@@ -200,7 +199,6 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CLIENT_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CONSISTENCY_CHECK_SKIPPED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DAEMON;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_GRID_NAME;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IPS;
@@ -209,11 +207,12 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JVM_ARGS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JVM_PID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LANG_RUNTIME;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM;
@@ -292,6 +291,14 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     @GridToStringExclude
     private ObjectName p2PExecSvcMBean;
 
+    /** */
+    @GridToStringExclude
+    private ObjectName utilityExecSvcMBean;
+
+    /** */
+    @GridToStringExclude
+    private ObjectName marshallerExecSvcMBean;
+
     /** */
     @GridToStringExclude
     private ObjectName restExecSvcMBean;
@@ -969,7 +976,14 @@ public void start(final IgniteConfiguration cfg,
             // Register MBeans.
             registerKernalMBean();
             registerLocalNodeMBean();
-            registerExecutorMBeans(execSvc, sysExecSvc, p2pExecSvc, mgmtExecSvc, restExecSvc);
+            registerExecutorMBeans(
+                execSvc,
+                sysExecSvc,
+                p2pExecSvc,
+                mgmtExecSvc,
+                restExecSvc,
+                utilityCachePool,
+                marshCachePool);
 
             // Lifecycle bean notifications.
             notifyLifecycleBeans(AFTER_NODE_START);
@@ -1523,15 +1537,21 @@ private void registerLocalNodeMBean() throws IgniteCheckedException {
     }
 
     /** @throws IgniteCheckedException If registration failed. */
-    private void registerExecutorMBeans(ExecutorService execSvc,
+    private void registerExecutorMBeans(
+        ExecutorService execSvc,
         ExecutorService sysExecSvc,
         ExecutorService p2pExecSvc,
         ExecutorService mgmtExecSvc,
-        ExecutorService restExecSvc) throws IgniteCheckedException {
+        ExecutorService restExecSvc,
+        ExecutorService utilityExecSvc,
+        ExecutorService marshallerExecSvc
+    ) throws IgniteCheckedException {
         pubExecSvcMBean = registerExecutorMBean(execSvc, "GridExecutionExecutor");
         sysExecSvcMBean = registerExecutorMBean(sysExecSvc, "GridSystemExecutor");
         mgmtExecSvcMBean = registerExecutorMBean(mgmtExecSvc, "GridManagementExecutor");
         p2PExecSvcMBean = registerExecutorMBean(p2pExecSvc, "GridClassLoadingExecutor");
+        utilityExecSvcMBean = registerExecutorMBean(utilityExecSvc ,"GridUtilityCacheExecutor");
+        marshallerExecSvcMBean = registerExecutorMBean(marshallerExecSvc, "GridMarshallerCacheExecutor");
 
         ConnectorConfiguration clientCfg = cfg.getConnectorConfiguration();
 
@@ -2027,6 +2047,8 @@ else if (state == STARTING)
                     unregisterMBean(p2PExecSvcMBean) &
                     unregisterMBean(kernalMBean) &
                     unregisterMBean(locNodeMBean) &
+                    unregisterMBean(utilityExecSvcMBean) &
+                    unregisterMBean(marshallerExecSvcMBean) &
                     unregisterMBean(restExecSvcMBean)
             ))
                 errOnStop = false;

From ed34a5dc681ea8f284f4d25c5575ad46569cc600 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Jun 2017 18:33:55 +0300
Subject: [PATCH 229/516] Partial fix of IGNITE-5562.

---
 .../spi/discovery/tcp/internal/TcpDiscoveryStatistics.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
index c79064491bb43..a69dbd9aa89aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
@@ -316,7 +316,9 @@ public synchronized void onMessageProcessingFinished(TcpDiscoveryAbstractMessage
      */
     public synchronized void onMessageSent(TcpDiscoveryAbstractMessage msg, long time, long ackTime) {
         assert msg != null;
-        assert time >= 0 : time;
+
+        if (time < 0)
+            time = 0;
 
         if (crdSinceTs.get() > 0 &&
             (msg instanceof TcpDiscoveryCustomEventMessage) ||

From d427021f329292fb69d348ba949ad1f8f1e9089e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 21 Jun 2017 19:30:27 +0300
Subject: [PATCH 230/516] IGNITE-5552: ServiceProcessor recalculates all
 service assignments even if there is a pending topology change.

---
 .../service/GridServiceProcessor.java         | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 25a8edbfc7c17..9c746401b1479 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -169,7 +169,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
     private IgniteInternalCache<Object, Object> cache;
 
     /** Topology listener. */
-    private GridLocalEventListener topLsnr = new TopologyListener();
+    private final GridLocalEventListener topLsnr = new TopologyListener();
 
     static {
         Set<IgniteProductVersion> versions = new TreeSet<>(new Comparator<IgniteProductVersion>() {
@@ -1574,6 +1574,9 @@ private void onDeployment(final GridServiceDeployment dep, final AffinityTopolog
      * Topology listener.
      */
     private class TopologyListener implements GridLocalEventListener {
+        /** */
+        private volatile AffinityTopologyVersion currTopVer = null;
+
         /** {@inheritDoc} */
         @Override public void onEvent(Event evt) {
             if (!busyLock.enterBusy())
@@ -1601,6 +1604,8 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                 else
                     topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
 
+                currTopVer = topVer;
+
                 depExe.submit(new DepRunnable() {
                     @Override public void run0() {
                         ClusterNode oldest = ctx.discovery().oldestAliveCacheServerNode(topVer);
@@ -1618,6 +1623,19 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                                 boolean firstTime = true;
 
                                 while (it.hasNext()) {
+                                    // If topology changed again, let next event handle it.
+                                    AffinityTopologyVersion currTopVer0 = currTopVer;
+
+                                    if (currTopVer0 != topVer) {
+                                        if (log.isInfoEnabled())
+                                            log.info("Service processor detected a topology change during " +
+                                                "assignments calculation (will abort current iteration and " +
+                                                "re-calculate on the newer version): " +
+                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer + ']');
+
+                                        return;
+                                    }
+
                                     Cache.Entry<Object, Object> e = it.next();
 
                                     if (!(e.getKey() instanceof GridServiceDeploymentKey))

From f1b9cdc0716a1b23f54d68ce0fe19eb85107567d Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Wed, 14 Jun 2017 21:37:54 +0300
Subject: [PATCH 231/516] GG-12354: Partial fix of IGNITE-5473: Introduce
 troubleshooting logger.

---
 .../dht/GridDhtPartitionTopologyImpl.java     |  2 +-
 ...NearAtomicAbstractSingleUpdateRequest.java |  8 +++
 .../GridNearAtomicAbstractUpdateFuture.java   |  8 +++
 ...idNearAtomicSingleUpdateInvokeRequest.java |  6 ++
 .../atomic/GridNearAtomicUpdateFuture.java    |  1 +
 .../atomic/GridNearAtomicUpdateResponse.java  |  2 +-
 .../service/GridServiceProcessor.java         | 49 ++++++++++---
 .../service/ServiceContextImpl.java           |  2 +
 .../util/tostring/GridToStringBuilder.java    |  2 +-
 .../tcp/TcpCommunicationSpi.java              | 57 +++++++--------
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 72 +++++++++++++------
 .../spi/discovery/tcp/TcpDiscoverySpi.java    | 16 +++--
 12 files changed, 157 insertions(+), 68 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 1b4dcc9293dee..1ad0ff0538fa5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -210,7 +210,7 @@ private boolean waitForRent() throws IgniteCheckedException {
 
                             break;
                         }
-                        catch (IgniteFutureTimeoutCheckedException e) {
+                        catch (IgniteFutureTimeoutCheckedException ignore) {
                             if (dumpCnt++ < GridDhtPartitionsExchangeFuture.DUMP_PENDING_OBJECTS_THRESHOLD) {
                                 U.warn(log, "Failed to wait for partition eviction [" +
                                     "topVer=" + topVer +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractSingleUpdateRequest.java
index 61deeee1e1042..1e87e251a6680 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractSingleUpdateRequest.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
 import org.apache.ignite.plugin.extensions.communication.MessageWriter;
 import org.jetbrains.annotations.NotNull;
@@ -559,4 +560,11 @@ private boolean isFlag(int mask) {
     @Override public byte fieldsCount() {
         return 11;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(GridNearAtomicAbstractSingleUpdateRequest.class, this,
+            "nodeId", nodeId, "futVer", futVer, "topVer", topVer,
+            "parent", super.toString());
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
index c92e0f5ec5b42..936c8a3bd3c87 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
@@ -342,4 +343,11 @@ protected final GridCacheVersion addAtomicFuture(AffinityTopologyVersion topVer)
 
         return futVer;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(GridNearAtomicAbstractUpdateFuture.class, this,
+            "topLocked", topLocked, "remapCnt", remapCnt, "resCnt", resCnt, "err", err,
+            "parent", super.toString());
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
index df9e38441e79f..bf6bc17f16732 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
@@ -303,4 +304,9 @@ public GridNearAtomicSingleUpdateInvokeRequest() {
     @Override public byte directType() {
         return 126;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(GridNearAtomicSingleUpdateRequest.class, this, super.toString());
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index 950e5bdd2b9b1..a252d9ae30c2b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -90,6 +90,7 @@ public class GridNearAtomicUpdateFuture extends GridNearAtomicAbstractUpdateFutu
     private Map<UUID, GridNearAtomicFullUpdateRequest> mappings;
 
     /** Keys to remap. */
+    @GridToStringInclude
     private Collection<KeyCacheObject> remapKeys;
 
     /** Not null is operation is mapped to single node. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
index 2e38733e8b900..e48b7265f51c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
@@ -642,6 +642,6 @@ public synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwabl
 
     /** {@inheritDoc} */
     @Override public String toString() {
-        return S.toString(GridNearAtomicUpdateResponse.class, this, "parent");
+        return S.toString(GridNearAtomicUpdateResponse.class, this, super.toString());
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 9c746401b1479..4ecf674a48420 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -974,12 +974,24 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
                         int cnt = maxPerNodeCnt == 0 ? totalCnt == 0 ? 1 : totalCnt : maxPerNodeCnt;
 
                         cnts.put(n.id(), cnt);
+
+                        if (log.isInfoEnabled())
+                            log.info("Assigned service to primary node [svc=" + dep.configuration().getName() +
+                                ", topVer=" + topVer + ", node=" + n.id() + ']');
                     }
                 }
                 else {
                     if (!nodes.isEmpty()) {
                         int size = nodes.size();
 
+                        if (log.isInfoEnabled())
+                            log.info("Calculating assignments for service " +
+                                "[svc=" + dep.configuration().getName() +
+                                ", topVer=" + topVer +
+                                ", nodes=" + U.nodeIds(nodes) +
+                                ", oldAssignment=" + (oldAssigns == null ? "NA" : oldAssigns.assigns()) +
+                                ", totalCnt=" + totalCnt + ", maxPerNodeCnt=" + maxPerNodeCnt + ']');
+
                         int perNodeCnt = totalCnt != 0 ? totalCnt / size : maxPerNodeCnt;
                         int remainder = totalCnt != 0 ? totalCnt % size : 0;
 
@@ -1055,6 +1067,10 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
 
                 assigns.assigns(cnts);
 
+                if (log.isInfoEnabled())
+                    log.info("Calculated new assignments for service [svc=" + dep.configuration().getName() +
+                        ", assignment=" + assigns + ']');
+
                 cache.put(key, assigns);
 
                 tx.commit();
@@ -1095,6 +1111,10 @@ private void redeploy(GridServiceAssignments assigns) {
         Collection<ServiceContextImpl> toInit = new ArrayList<>();
 
         synchronized (ctxs) {
+            if (log.isInfoEnabled())
+                log.info("Updating service deployment [locNodeId=" + ctx.localNodeId() +
+                    ", svc=" + assigns.name() + ", assigns=" + assigns + ']');
+
             if (ctxs.size() > assignCnt) {
                 int cancelCnt = ctxs.size() - assignCnt;
 
@@ -1483,6 +1503,10 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
         }
         // Handle undeployment.
         else {
+            if (log.isInfoEnabled())
+                log.info("Received undeploy notification, will cancel all instances of the service " +
+                    "[svc=" + e.getKey().name() + "]");
+
             String name = e.getKey().name();
 
             undeploy(name);
@@ -1524,13 +1548,15 @@ private void onDeployment(final GridServiceDeployment dep, final AffinityTopolog
         try {
             AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
-            // If topology version changed, reassignment will happen from topology event.
-            if (newTopVer.equals(topVer))
-                reassign(dep, topVer);
-        }
-        catch (IgniteCheckedException e) {
-            if (!(e instanceof ClusterTopologyCheckedException))
-                log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
+                // If topology version changed, reassignment will happen from topology event.
+                if (newTopVer.equals(topVer)){
+                    if (log.isInfoEnabled())
+                        log.info("Will calculate new service deployment assignment (new service has been deployed) [" +
+                            "svc=" + dep.configuration().getName() + ", topVer=" + topVer + ']');reassign(dep, topVer);}
+            }
+            catch (IgniteCheckedException e) {
+                if (!(e instanceof ClusterTopologyCheckedException))
+                    log.error("Failed to do service reassignment (will retry): " + dep.configuration().getName(), e);
 
             AffinityTopologyVersion newTopVer = ctx.discovery().topologyVersionEx();
 
@@ -1817,9 +1843,12 @@ private void undeploy(String name) {
             ctxs = locSvcs.remove(name);
         }
 
-        if (ctxs != null) {
-            synchronized (ctxs) {
-                cancel(ctxs, ctxs.size());
+            if (ctxs != null) {
+                synchronized (ctxs) {
+                    if (log.isInfoEnabled())
+                        log.info("Undeploying services [svc=" + e.getKey().name() + ", assigns=" + assigns +
+                            ", ctxs=" + ctxs + ']');cancel(ctxs, ctxs.size());
+
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceContextImpl.java
index 66061d5c6f13e..490b5986faff9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceContextImpl.java
@@ -23,6 +23,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceContext;
@@ -48,6 +49,7 @@ public class ServiceContextImpl implements ServiceContext {
     private final String cacheName;
 
     /** Affinity key. */
+    @GridToStringInclude
     private final Object affKey;
 
     /** Executor service. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
index 6807b3f8ad0ed..e4c8c66c05580 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
@@ -87,7 +87,7 @@ public class GridToStringBuilder {
     private static final ReadWriteLock rwLock = new ReentrantReadWriteLock();
 
     /** Maximum number of collection (map) entries to print. */
-    public static final int MAX_COL_SIZE = 100;
+    public static final int MAX_COL_SIZE = 200;
 
     /** {@link IgniteSystemProperties#IGNITE_TO_STRING_INCLUDE_SENSITIVE} */
     public static final boolean INCLUDE_SENSITIVE =
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 6dba6b2b5fd0c..80cb77633b844 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -422,8 +422,8 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                 final ClusterNode rmtNode = getSpiContext().node(sndId);
 
                 if (rmtNode == null) {
-                    if (log.isDebugEnabled())
-                        log.debug("Close incoming connection, unknown node: " + sndId);
+                    U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId +
+                        ", ses=" + ses + ']');
 
                     ses.close();
 
@@ -441,10 +441,10 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
 
                 if (oldClient != null) {
                     if (oldClient instanceof GridTcpNioCommunicationClient) {
-                        if (log.isDebugEnabled())
-                            log.debug("Received incoming connection when already connected " +
-                                    "to this node, rejecting [locNode=" + locNode.id() +
-                                    ", rmtNode=" + sndId + ']');
+                        if (log.isInfoEnabled())
+                            log.info("Received incoming connection when already connected " +
+                                "to this node, rejecting [locNode=" + locNode.id() +
+                                ", rmtNode=" + sndId + ']');
 
                         ses.send(new RecoveryLastReceivedMessage(-1));
 
@@ -472,10 +472,10 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
 
                     if (oldClient != null) {
                         if (oldClient instanceof GridTcpNioCommunicationClient) {
-                            if (log.isDebugEnabled())
-                                log.debug("Received incoming connection when already connected " +
-                                        "to this node, rejecting [locNode=" + locNode.id() +
-                                        ", rmtNode=" + sndId + ']');
+                            if (log.isInfoEnabled())
+                                log.info("Received incoming connection when already connected " +
+                                    "to this node, rejecting [locNode=" + locNode.id() +
+                                    ", rmtNode=" + sndId + ']');
 
                             ses.send(new RecoveryLastReceivedMessage(-1));
 
@@ -495,7 +495,7 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
 
                     if (log.isDebugEnabled())
                         log.debug("Received incoming connection from remote node " +
-                                "[rmtNode=" + rmtNode.id() + ", reserved=" + reserved + ']');
+                            "[rmtNode=" + rmtNode.id() + ", reserved=" + reserved + ']');
 
                     if (reserved) {
                         try {
@@ -511,11 +511,11 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                 }
                 else {
                     if (oldFut instanceof ConnectFuture && locNode.order() < rmtNode.order()) {
-                        if (log.isDebugEnabled()) {
-                            log.debug("Received incoming connection from remote node while " +
-                                    "connecting to this node, rejecting [locNode=" + locNode.id() +
-                                    ", locNodeOrder=" + locNode.order() + ", rmtNode=" + rmtNode.id() +
-                                    ", rmtNodeOrder=" + rmtNode.order() + ']');
+                        if (log.isInfoEnabled()) {
+                            log.info("Received incoming connection from remote node while " +
+                                "connecting to this node, rejecting [locNode=" + locNode.id() +
+                                ", locNodeOrder=" + locNode.order() + ", rmtNode=" + rmtNode.id() +
+                                ", rmtNodeOrder=" + rmtNode.order() + ']');
                         }
 
                         ses.send(new RecoveryLastReceivedMessage(-1));
@@ -2502,11 +2502,11 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                                 ", addr=" + addr + ", err=" + e + ']');
 
                     if (attempt == reconCnt || connTimeout0 > maxConnTimeout) {
-                        if (log.isDebugEnabled())
-                            log.debug("Handshake timedout (will stop attempts to perform the handshake) " +
-                                "[timeout=" + connTimeout0 + ", maxConnTimeout=" + maxConnTimeout +
-                                ", attempt=" + attempt + ", reconCnt=" + reconCnt +
-                                ", err=" + e.getMessage() + ", addr=" + addr + ']');
+                        U.warn(log, "Handshake timedout (will stop attempts to perform the handshake) " +
+                            "[node=" + node.id() + ", timeout=" + connTimeout0 +
+                            ", maxConnTimeout=" + maxConnTimeout +
+                            ", attempt=" + attempt + ", reconCnt=" + reconCnt +
+                            ", err=" + e.getMessage() + ", addr=" + addr + ']');
 
                         if (errs == null)
                             errs = new IgniteCheckedException("Failed to connect to node (is node still alive?). " +
@@ -2540,6 +2540,10 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
 
                     boolean failureDetThrReached = timeoutHelper.checkFailureTimeoutReached(e);
 
+                    U.error(log, "Failed to establish connection to a remote node [node=" + node +
+                        ", addr=" + addr + ", connectAttempts=" + connectAttempts +
+                        ", failureDetThrReached" + failureDetThrReached + ']', e);
+
                     if (failureDetThrReached)
                         LT.warn(log, "Connect timed out (consider increasing 'failureDetectionTimeout' " +
                             "configuration property) [addr=" + addr + ", failureDetectionTimeout=" +
@@ -2582,13 +2586,10 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     "[addrs=" + addrs + ']');
 
             if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
-                X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
-                    IgniteSpiOperationTimeoutException.class)) {
-                LT.warn(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
-                    "cluster [" +
-                    "rmtNode=" + node +
-                    ", err=" + errs +
-                    ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
+                    X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
+                        IgniteSpiOperationTimeoutException.class)) {
+                    U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
+                        "cluster [" + "rmtNode=" + node + ']', errs);
 
                 getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
                     "rmtNode=" + node +
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 6bdd80cbe6ff6..2f7e9b470aae6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -514,9 +514,6 @@ else if (log.isInfoEnabled()) {
     @Override public boolean pingNode(UUID nodeId) {
         assert nodeId != null;
 
-        if (log.isDebugEnabled())
-            log.debug("Pinging node: " + nodeId + "]");
-
         if (nodeId == getLocalNodeId())
             return true;
 
@@ -528,8 +525,18 @@ else if (log.isInfoEnabled()) {
         if (!nodeAlive(nodeId))
             return false;
 
+        long start = U.currentTimeMillis();
+
+        if (log.isInfoEnabled())
+            log.info("Pinging node: " + nodeId);
+
         boolean res = pingNode(node);
 
+        long end = System.currentTimeMillis();
+
+        if (log.isInfoEnabled())
+            log.info("Finished node ping [nodeId=" + nodeId + ", res=" + res + ", time=" + (end - start) + "ms]");
+
         if (!res && !node.isClient() && nodeAlive(nodeId)) {
             LT.warn(log, "Failed to ping node (status check will be initiated): " + nodeId);
 
@@ -1686,9 +1693,9 @@ TcpDiscoveryNodesRing ring() {
 
         assert log.isInfoEnabled();
 
-        synchronized (mux) {
-            StringBuilder b = new StringBuilder(U.nl());
+        StringBuilder b = new StringBuilder(U.nl());
 
+        synchronized (mux) {
             b.append(">>>").append(U.nl());
             b.append(">>>").append("Dumping discovery SPI debug info.").append(U.nl());
             b.append(">>>").append(U.nl());
@@ -1735,9 +1742,9 @@ TcpDiscoveryNodesRing ring() {
             b.append(U.nl());
 
             b.append("Stats: ").append(spi.stats).append(U.nl());
-
-            U.quietAndInfo(log, b.toString());
         }
+
+        U.quietAndInfo(log, b.toString());
     }
 
     /**
@@ -3171,12 +3178,8 @@ else if (e instanceof SocketTimeoutException ||
 
                             // If node existed on connection initialization we should check
                             // whether it has not gone yet.
-                            if (nextNodeExists)
-                                U.warn(log, "Failed to send message to next node [msg=" + msg + ", next=" + next +
-                                    ", errMsg=" + (err != null ? err.getMessage() : "N/A") + ']');
-                            else if (log.isDebugEnabled())
-                                log.debug("Failed to send message to next node [msg=" + msg + ", next=" + next +
-                                    ", errMsg=" + (err != null ? err.getMessage() : "N/A") + ']');
+                            U.warn(log, "Failed to send message to next node [msg=" + msg + ", next=" + next +
+                                ", errMsg=" + (err != null ? err.getMessage() : "N/A") + ']');
                         }
                     }
 
@@ -3358,6 +3361,8 @@ private void processJoinRequestMessage(final TcpDiscoveryJoinRequestMessage msg)
                 if (existingNode != null) {
                     if (!node.socketAddresses().equals(existingNode.socketAddresses())) {
                         if (!pingNode(existingNode)) {
+                            U.warn(log, "Sending node failed message for existing node: " + node);
+
                             addMessage(new TcpDiscoveryNodeFailedMessage(locNodeId,
                                 existingNode.id(), existingNode.internalOrder()));
 
@@ -5421,8 +5426,7 @@ private void checkFailedNodesList() {
 
             if (msgs != null) {
                 for (TcpDiscoveryNodeFailedMessage msg : msgs) {
-                    if (log.isDebugEnabled())
-                        log.debug("Add node failed message for node from failed nodes list: " + msg);
+                    U.warn(log, "Added node failed message for node from failed nodes list: " + msg);
 
                     addMessage(msg);
                 }
@@ -5641,8 +5645,9 @@ private class TcpServer extends IgniteSpiThread {
 
                     long tstamp = U.currentTimeMillis();
 
-                    if (log.isDebugEnabled())
-                        log.debug("Accepted incoming connection from addr: " + sock.getInetAddress());
+                    if (log.isInfoEnabled())
+                        log.info("TCP discovery accepted incoming connection " +
+                            "[rmtAddr=" + sock.getInetAddress() + ", rmtPort=" + sock.getPort() + ']');
 
                     SocketReader reader = new SocketReader(sock);
 
@@ -5650,6 +5655,10 @@ private class TcpServer extends IgniteSpiThread {
                         readers.add(reader);
                     }
 
+                    if (log.isInfoEnabled())
+                        log.info("TCP discovery spawning a new thread for connection " +
+                            "[rmtAddr=" + sock.getInetAddress() + ", rmtPort=" + sock.getPort() + ']');
+
                     reader.start();
 
                     spi.stats.onServerSocketInitialized(U.currentTimeMillis() - tstamp);
@@ -5712,6 +5721,12 @@ private class SocketReader extends IgniteSpiThread {
 
             ClientMessageWorker clientMsgWrk = null;
 
+            SocketAddress rmtAddr = sock.getRemoteSocketAddress();
+
+            if (log.isInfoEnabled())
+                log.info("Started serving remote node connection [rmtAddr=" + rmtAddr +
+                    ", rmtPort=" + sock.getPort() + ']');
+
             try {
                 InputStream in;
 
@@ -5742,11 +5757,11 @@ private class SocketReader extends IgniteSpiThread {
                         else {
                             if (log.isDebugEnabled())
                                 log.debug("Failed to read magic header (too few bytes received) " +
-                                    "[rmtAddr=" + sock.getRemoteSocketAddress() +
+                                    "[rmtAddr=" + rmtAddr +
                                     ", locAddr=" + sock.getLocalSocketAddress() + ']');
 
                             LT.warn(log, "Failed to read magic header (too few bytes received) [rmtAddr=" +
-                                sock.getRemoteSocketAddress() + ", locAddr=" + sock.getLocalSocketAddress() + ']');
+                                rmtAddr + ", locAddr=" + sock.getLocalSocketAddress() + ']');
 
                             return;
                         }
@@ -5758,7 +5773,7 @@ private class SocketReader extends IgniteSpiThread {
                                 "this Ignite port?" +
                                 (!spi.isSslEnabled() ? " missed SSL configuration?" : "" ) +
                                 ") " +
-                                "[rmtAddr=" + sock.getRemoteSocketAddress() +
+                                "[rmtAddr=" + rmtAddr +
                                 ", locAddr=" + sock.getLocalSocketAddress() + ']');
 
                         LT.warn(log, "Unknown connection detected (is some other software connecting to " +
@@ -5779,6 +5794,11 @@ private class SocketReader extends IgniteSpiThread {
                         if (!spi.isNodeStopping0()) {
                             TcpDiscoveryPingRequest req = (TcpDiscoveryPingRequest)msg;
 
+                            if (log.isInfoEnabled())
+                                log.info("Received ping request from the remote node " +
+                                    "[rmtNodeId=" + msg.creatorNodeId() +
+                                    ", rmtAddr=" + rmtAddr + ", rmtPort=" + sock.getPort() + "]");
+
                             TcpDiscoveryPingResponse res = new TcpDiscoveryPingResponse(locNodeId);
 
                             IgniteSpiOperationTimeoutHelper timeoutHelper =
@@ -5792,6 +5812,12 @@ private class SocketReader extends IgniteSpiThread {
                             }
 
                             spi.writeToSocket(sock, res, timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()));
+
+                            sock.shutdownOutput();
+
+                            if (log.isInfoEnabled())
+                                log.info("Finished writing ping response " + "[rmtNodeId=" + msg.creatorNodeId() +
+                                    ", rmtAddr=" + rmtAddr + ", rmtPort=" + sock.getPort() + "]");
                         }
                         else if (log.isDebugEnabled())
                             log.debug("Ignore ping request, node is stopping.");
@@ -5917,7 +5943,7 @@ else if ((X.hasCause(e, ObjectStreamException.class) || !sock.isClosed())
                     else if (e.hasCause(ClassNotFoundException.class))
                         LT.warn(log, "Failed to read message due to ClassNotFoundException " +
                             "(make sure same versions of all classes are available on all nodes) " +
-                            "[rmtAddr=" + sock.getRemoteSocketAddress() +
+                            "[rmtAddr=" + rmtAddr +
                             ", err=" + X.cause(e, ClassNotFoundException.class).getMessage() + ']');
 
                         // Always report marshalling problems.
@@ -6208,6 +6234,10 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage) {
                 }
 
                 U.closeQuiet(sock);
+
+                if (log.isInfoEnabled())
+                    log.info("Finished serving remote node connection [rmtAddr=" + rmtAddr +
+                        ", rmtPort=" + sock.getPort());
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index cefbcd675a303..3cc4ee53242b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1482,7 +1482,8 @@ protected <T> T readMessage(Socket sock, @Nullable InputStream in, long timeout)
         catch (IOException | IgniteCheckedException e) {
             if (X.hasCause(e, SocketTimeoutException.class))
                 LT.warn(log, "Timed out waiting for message to be read (most probably, the reason is " +
-                    "in long GC pauses on remote node) [curTimeout=" + timeout + ']');
+                    "long GC pauses on remote node) [curTimeout=" + timeout +
+                    ", rmtAddr=" + sock.getRemoteSocketAddress() + ", rmtPort=" + sock.getPort() + ']');
 
             throw e;
         }
@@ -1523,8 +1524,9 @@ protected int readReceipt(Socket sock, long timeout) throws IOException {
         catch (SocketTimeoutException e) {
             LT.warn(log, "Timed out waiting for message delivery receipt (most probably, the reason is " +
                 "in long GC pauses on remote node; consider tuning GC and increasing 'ackTimeout' " +
-                "configuration property). Will retry to send message with increased timeout. " +
-                "Current timeout: " + timeout + '.');
+                "configuration property). Will retry to send message with increased timeout " +
+                "[currentTimeout=" + timeout + ", rmtAddr=" + sock.getRemoteSocketAddress() +
+                ", rmtPort=" + sock.getPort() + ']');
 
             stats.onAckTimeout();
 
@@ -2066,9 +2068,11 @@ boolean cancel() {
 
                 LT.warn(log, "Socket write has timed out (consider increasing " +
                     (failureDetectionTimeoutEnabled() ?
-                    "'IgniteConfiguration.failureDetectionTimeout' configuration property) [" +
-                    "failureDetectionTimeout=" + failureDetectionTimeout() + ']' :
-                    "'sockTimeout' configuration property) [sockTimeout=" + sockTimeout + ']'));
+                        "'IgniteConfiguration.failureDetectionTimeout' configuration property) [" +
+                        "failureDetectionTimeout=" + failureDetectionTimeout() :
+                        "'sockTimeout' configuration property) [sockTimeout=" + sockTimeout) +
+                    ", rmtAddr=" + sock.getRemoteSocketAddress() + ", rmtPort=" + sock.getPort() +
+                    ", sockTimeout=" + sockTimeout + ']');
 
                 stats.onSocketTimeout();
             }

From beb2409cfe2045789443d47de735d879961d371e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 23 Jun 2017 12:26:06 +0300
Subject: [PATCH 232/516] GG-12352: Forcible node drop makes cluster instable
 in some cases. Disable forcible node drop by default.

---
 .../apache/ignite/IgniteSystemProperties.java |  6 +++
 .../service/GridServiceProcessor.java         |  2 +-
 .../spi/IgniteSpiOperationTimeoutHelper.java  |  6 ++-
 .../tcp/TcpCommunicationSpi.java              | 46 +++++++++++++------
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  6 ++-
 .../tcp/TcpCommunicationSpiDropNodesTest.java | 16 +++++++
 .../TcpCommunicationSpiFaultyClientTest.java  | 15 ++++++
 7 files changed, 79 insertions(+), 18 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 775aae03a1ef4..f473c51e74a93 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -533,6 +533,12 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_SECURITY_COMPATIBILITY_MODE = "IGNITE_SECURITY_COMPATIBILITY_MODE";
 
+    /**
+     * If this property is set, a node will forcible fail a remote node when it fails to establish a communication
+     * connection.
+     */
+    public static final String IGNITE_ENABLE_FORCIBLE_NODE_KILL = "IGNITE_ENABLE_FORCIBLE_NODE_KILL";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 4ecf674a48420..75c79a3a31fdc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1604,7 +1604,7 @@ private class TopologyListener implements GridLocalEventListener {
         private volatile AffinityTopologyVersion currTopVer = null;
 
         /** {@inheritDoc} */
-        @Override public void onEvent(Event evt) {
+        @Override public void onEvent(final Event evt) {
             if (!busyLock.enterBusy())
                 return;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
index e17b0dd1b196f..1d9fa94f44475 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
@@ -96,7 +96,9 @@ public boolean checkFailureTimeoutReached(Exception e) {
         if (!failureDetectionTimeoutEnabled)
             return false;
 
-        return e instanceof IgniteSpiOperationTimeoutException || e instanceof SocketTimeoutException ||
-            X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketException.class);
+        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class))
+            return true;
+
+        return X.hasCause(e, SocketException.class) && (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 80cb77633b844..98183d8afbd7a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -23,6 +23,7 @@
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -53,6 +54,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.AddressResolver;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -328,6 +330,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     /** */
     private ConnectGateway connectGate;
 
+    /** */
+    private boolean enableForcibleNodeKill = IgniteSystemProperties
+        .getBoolean(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
     /** Server listener. */
     private final GridNioServerListener<Message> srvLsnr =
         new GridNioServerListenerAdapter<Message>() {
@@ -2073,8 +2079,13 @@ private GridCommunicationClient reserveClient(ClusterNode node) throws IgniteChe
                                     }
                                 }
                             }
-                            else
+                            else {
                                 U.sleep(200);
+
+                                if (getSpiContext().node(node.id()) == null)
+                                    throw new ClusterTopologyCheckedException("Failed to send message " +
+                                        "(node left topology): " + node);
+                            }
                         }
 
                         fut.onDone(client0);
@@ -2470,6 +2481,7 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
+                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +
@@ -2561,8 +2573,10 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     errs.addSuppressed(new IgniteCheckedException("Failed to connect to address: " + addr, e));
 
                     // Reconnect for the second time, if connection is not established.
-                    if (!failureDetThrReached && connectAttempts < 2 &&
-                        (e instanceof ConnectException || X.hasCause(e, ConnectException.class))) {
+                    if (!failureDetThrReached && connectAttempts < 5 &&
+                        (X.hasCause(e, ConnectException.class) || X.hasCause(e, SocketTimeoutException.class))) {
+                        U.sleep(200);
+
                         connectAttempts++;
 
                         continue;
@@ -2585,19 +2599,22 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     "operating system firewall is disabled on local and remote hosts) " +
                     "[addrs=" + addrs + ']');
 
-            if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
+            if (enableForcibleNodeKill) {
+                if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
                     X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
                         IgniteSpiOperationTimeoutException.class)) {
                     U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
                         "cluster [" + "rmtNode=" + node + ']', errs);
 
-                getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
-                    "rmtNode=" + node +
-                    ", errs=" + errs +
-                    ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
+                    getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
+                        "rmtNode=" + node +
+                        ", errs=" + errs +
+                        ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
+                }
             }
 
-            throw errs;
+            if (X.hasCause(errs, ConnectException.class))
+                throw errs;
         }
 
         return client;
@@ -2829,8 +2846,9 @@ else if (log.isDebugEnabled())
 
             // Ignoring whatever happened after timeout - reporting only timeout event.
             if (!cancelled)
-                throw new HandshakeTimeoutException("Failed to perform handshake due to timeout (consider increasing " +
-                    "'connectionTimeout' configuration property).");
+                throw new HandshakeTimeoutException(
+                    new IgniteSpiOperationTimeoutException("Failed to perform handshake due to timeout " +
+                        "(consider increasing 'connectionTimeout' configuration property)."));
         }
 
         return rcvCnt;
@@ -2980,10 +2998,10 @@ private static class HandshakeTimeoutException extends IgniteCheckedException {
         private static final long serialVersionUID = 0L;
 
         /**
-         * @param msg Message.
+         * @param cause Exception cause
          */
-        HandshakeTimeoutException(String msg) {
-            super(msg);
+        HandshakeTimeoutException(IgniteSpiOperationTimeoutException cause) {
+            super(cause);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 2f7e9b470aae6..856370c2a0ee3 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1146,6 +1146,8 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
             boolean openSock = false;
 
+            boolean wasHandshake = false;
+
             Socket sock = null;
 
             try {
@@ -1163,6 +1165,8 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(
                     ackTimeout0));
 
+                wasHandshake = true;
+
                 if (msg instanceof TcpDiscoveryJoinRequestMessage) {
                     boolean ignore = false;
 
@@ -1264,7 +1268,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 if (!spi.failureDetectionTimeoutEnabled() && ++reconCnt == spi.getReconnectCount())
                     break;
 
-                if (!openSock) {
+                if (!openSock || !wasHandshake) {
                     // Reconnect for the second time, if connection is not established.
                     if (connectAttempts < 2) {
                         connectAttempts++;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
index d29231e449b0b..7b438d6795a8b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
@@ -25,6 +25,7 @@
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.TimeUnit;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
@@ -41,6 +42,7 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.config.GridTestProperties;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
@@ -82,6 +84,20 @@ public class TcpCommunicationSpiDropNodesTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
index 6e99487bd46ae..33625e4b9edb4 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
@@ -95,6 +96,20 @@ public class TcpCommunicationSpiFaultyClientTest extends GridCommonAbstractTest
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();

From 802f18fc250cbae8959192c78bb28dc525ed3cf7 Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Fri, 23 Jun 2017 00:24:57 +0300
Subject: [PATCH 233/516] Fix compilation

---
 .../internal/processors/service/GridServiceProcessor.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 75c79a3a31fdc..009bf47c57101 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1846,7 +1846,7 @@ private void undeploy(String name) {
             if (ctxs != null) {
                 synchronized (ctxs) {
                     if (log.isInfoEnabled())
-                        log.info("Undeploying services [svc=" + e.getKey().name() + ", assigns=" + assigns +
+                        log.info("Undeploying services [svc=" + name +
                             ", ctxs=" + ctxs + ']');cancel(ctxs, ctxs.size());
 
             }

From a64339449be8fa602cab3f2868c5f74004a7b747 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Fri, 23 Jun 2017 16:57:49 +0300
Subject: [PATCH 234/516] IGNITE-4370: Implemented writing of batch of
 parameters for ODBC.

(cherry picked from commit c10be5780589cc84e7929e234e4411d515166e0b)
(cherry picked from commit d268b32cb252a5f06887d2b803d27ddc20ded95f)
---
 .../processors/odbc/OdbcMessageParser.java    |  57 ++-
 .../processors/odbc/OdbcNioListener.java      |   3 +-
 .../odbc/OdbcQueryExecuteBatchRequest.java    |  95 ++++
 .../odbc/OdbcQueryExecuteBatchResult.java     |  75 ++++
 .../odbc/OdbcQueryExecuteRequest.java         |   6 +-
 .../odbc/OdbcQueryExecuteResult.java          |   2 +-
 .../internal/processors/odbc/OdbcRequest.java |   3 +
 .../processors/odbc/OdbcRequestHandler.java   | 106 ++++-
 .../src/application_data_buffer_test.cpp      | 146 ++-----
 .../cpp/odbc-test/src/column_test.cpp         |  21 +-
 .../cpp/odbc-test/src/queries_test.cpp        | 386 ++++++++++++++++-
 .../platforms/cpp/odbc-test/src/row_test.cpp  |   9 +-
 modules/platforms/cpp/odbc/Makefile.am        |   3 +
 .../platforms/cpp/odbc/include/Makefile.am    |   2 +
 .../ignite/odbc/app/application_data_buffer.h |  38 +-
 .../odbc/include/ignite/odbc/app/parameter.h  |  20 +-
 .../include/ignite/odbc/app/parameter_set.h   | 268 ++++++++++++
 .../odbc/include/ignite/odbc/common_types.h   |   4 +-
 .../ignite/odbc/diagnostic/diagnosable.h      |   4 +-
 .../cpp/odbc/include/ignite/odbc/message.h    | 408 +++++++-----------
 .../include/ignite/odbc/query/batch_query.h   | 160 +++++++
 .../include/ignite/odbc/query/data_query.h    |  20 +-
 .../odbc/include/ignite/odbc/query/query.h    |   3 +
 .../ignite/odbc/query/type_info_query.h       |   2 +-
 .../cpp/odbc/include/ignite/odbc/statement.h  |  64 +--
 .../cpp/odbc/project/vs/odbc.vcxproj          |   5 +
 .../cpp/odbc/project/vs/odbc.vcxproj.filters  |  15 +
 .../odbc/src/app/application_data_buffer.cpp  |  90 +++-
 .../platforms/cpp/odbc/src/app/parameter.cpp  |  11 +-
 .../cpp/odbc/src/app/parameter_set.cpp        | 242 +++++++++++
 .../cpp/odbc/src/config/connection_info.cpp   |  36 ++
 modules/platforms/cpp/odbc/src/message.cpp    | 366 ++++++++++++++++
 modules/platforms/cpp/odbc/src/odbc.cpp       |  16 +-
 .../cpp/odbc/src/query/batch_query.cpp        | 198 +++++++++
 .../cpp/odbc/src/query/data_query.cpp         |  13 +-
 modules/platforms/cpp/odbc/src/statement.cpp  | 267 ++++++++----
 36 files changed, 2599 insertions(+), 565 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java
 create mode 100644 modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
 create mode 100644 modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
 create mode 100644 modules/platforms/cpp/odbc/src/app/parameter_set.cpp
 create mode 100644 modules/platforms/cpp/odbc/src/message.cpp
 create mode 100644 modules/platforms/cpp/odbc/src/query/batch_query.cpp

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java
index 7b863d6b2af21..0cb89d71f626b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcMessageParser.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
 
 import java.util.Collection;
+import org.jetbrains.annotations.NotNull;
 
 /**
  * ODBC message parser.
@@ -99,23 +100,37 @@ public OdbcRequest decode(byte[] msg) {
         }
 
         OdbcRequest res;
-
+        
         switch (cmd) {
             case OdbcRequest.EXECUTE_SQL_QUERY: {
                 String cache = reader.readString();
                 String sql = reader.readString();
-                int argsNum = reader.readInt();
-
-                Object[] params = new Object[argsNum];
+                int paramNum = reader.readInt();
 
-                for (int i = 0; i < argsNum; ++i)
-                    params[i] = reader.readObjectDetached();
+                Object[] params = readParameterRow(reader, paramNum);
 
                 res = new OdbcQueryExecuteRequest(cache, sql, params);
 
                 break;
             }
 
+            case OdbcRequest.QRY_EXEC_BATCH: {
+                String schema = reader.readString();
+                String sql = reader.readString();
+                int paramRowLen = reader.readInt();
+                int rowNum = reader.readInt();
+                boolean last = reader.readBoolean();
+
+                Object[][] params = new Object[rowNum][];
+
+                for (int i = 0; i < rowNum; ++i)
+                    params[i] = readParameterRow(reader, paramRowLen);
+
+                res = new OdbcQueryExecuteBatchRequest(schema, sql, last, params);
+
+                break;
+            }
+
             case OdbcRequest.FETCH_SQL_QUERY: {
                 long queryId = reader.readLong();
                 int pageSize = reader.readInt();
@@ -170,6 +185,21 @@ public OdbcRequest decode(byte[] msg) {
         return res;
     }
 
+    /**
+     * Read row of parameters using reader.
+     * @param reader reader
+     * @param paramNum Number of parameters in a row
+     * @return Parameters array.
+     */
+    @NotNull private static Object[] readParameterRow(BinaryReaderExImpl reader, int paramNum) {
+        Object[] params = new Object[paramNum];
+
+        for (int i = 0; i < paramNum; ++i)
+            params[i] = reader.readObjectDetached();
+
+        return params;
+    }
+
     /**
      * Encode OdbcResponse to byte array.
      *
@@ -218,9 +248,9 @@ else if (res0 instanceof OdbcQueryExecuteResult) {
             OdbcQueryExecuteResult res = (OdbcQueryExecuteResult) res0;
 
             if (log.isDebugEnabled())
-                log.debug("Resulting query ID: " + res.getQueryId());
+                log.debug("Resulting query ID: " + res.queryId());
 
-            writer.writeLong(res.getQueryId());
+            writer.writeLong(res.queryId());
 
             Collection<OdbcColumnMeta> metas = res.getColumnsMetadata();
 
@@ -231,6 +261,17 @@ else if (res0 instanceof OdbcQueryExecuteResult) {
             for (OdbcColumnMeta meta : metas)
                 meta.write(writer);
         }
+        else if (res0 instanceof OdbcQueryExecuteBatchResult) {
+            OdbcQueryExecuteBatchResult res = (OdbcQueryExecuteBatchResult) res0;
+
+            writer.writeBoolean(res.errorMessage() == null);
+            writer.writeLong(res.rowsAffected());
+
+            if (res.errorMessage() != null) {
+                writer.writeLong(res.errorSetIdx());
+                writer.writeString(res.errorMessage());
+            }
+        }
         else if (res0 instanceof OdbcQueryFetchResult) {
             OdbcQueryFetchResult res = (OdbcQueryFetchResult) res0;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java
index 378e5f25ebebc..a68cf888af48b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcNioListener.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.internal.util.nio.GridNioServerListenerAdapter;
 import org.apache.ignite.internal.util.nio.GridNioSession;
 import org.apache.ignite.internal.util.nio.GridNioSessionMetaKey;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.concurrent.atomic.AtomicLong;
@@ -100,7 +101,7 @@ public OdbcNioListener(GridKernalContext ctx, GridSpinBusyLock busyLock, int max
             req = parser.decode(msg);
         }
         catch (Exception e) {
-            log.error("Failed to parse message [id=" + reqId + ", err=" + e + ']');
+            U.error(log, "Failed to parse message [id=" + reqId + ']', e);
 
             ses.close();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java
new file mode 100644
index 0000000000000..0ace947126349
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchRequest.java
@@ -0,0 +1,95 @@
+package org.apache.ignite.internal.processors.odbc;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * ODBC query execute request with the batch of parameters.
+ */
+public class OdbcQueryExecuteBatchRequest extends OdbcRequest {
+    /** Schema. */
+    @GridToStringInclude(sensitive = true)
+    private final String schema;
+
+    /** Sql query. */
+    @GridToStringInclude(sensitive = true)
+    private final String sqlQry;
+
+    /** Last param page flag. */
+    private final boolean last;
+
+    /** Sql query arguments. */
+    @GridToStringExclude
+    private final Object[][] args;
+
+    /**
+     * @param schema Schema.
+     * @param sqlQry SQL query.
+     * @param last Last page flag.
+     * @param args Arguments list.
+     */
+    public OdbcQueryExecuteBatchRequest(@Nullable String schema, String sqlQry, boolean last, Object[][] args) {
+        super(QRY_EXEC_BATCH);
+
+        assert sqlQry != null : "SQL query should not be null";
+        assert args != null : "Parameters should not be null";
+
+        this.schema = schema;
+        this.sqlQry = sqlQry;
+        this.last = last;
+        this.args = args;
+    }
+
+    /**
+     * @return Sql query.
+     */
+    public String sqlQuery() {
+        return sqlQry;
+    }
+
+    /**
+     * @return Sql query arguments.
+     */
+    public Object[][] arguments() {
+        return args;
+    }
+
+    /**
+     * @return Schema.
+     */
+    @Nullable
+    public String schema() {
+        return schema;
+    }
+
+    /**
+     * @return Last page flag.
+     */
+    public boolean last() {
+        return last;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(OdbcQueryExecuteBatchRequest.class, this, "args", args, true);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java
new file mode 100644
index 0000000000000..6fc3873dea267
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteBatchResult.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.odbc;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * ODBC query execute with batch of parameters result.
+ */
+public class OdbcQueryExecuteBatchResult {
+    /** Rows affected. */
+    private final long rowsAffected;
+
+    /** Index of the set which caused an error. */
+    private final long errorSetIdx;
+
+    /** Error message. */
+    private final String errorMessage;
+
+    /**
+     * @param rowsAffected Number of rows affected by the query.
+     */
+    public OdbcQueryExecuteBatchResult(long rowsAffected) {
+        this.rowsAffected = rowsAffected;
+        this.errorSetIdx = -1;
+        this.errorMessage = null;
+    }
+
+    /**
+     * @param rowsAffected Number of rows affected by the query.
+     * @param errorSetIdx Sets processed.
+     * @param errorMessage Error message.
+     */
+    public OdbcQueryExecuteBatchResult(long rowsAffected, long errorSetIdx, String errorMessage) {
+        this.rowsAffected = rowsAffected;
+        this.errorSetIdx = errorSetIdx;
+        this.errorMessage = errorMessage;
+    }
+
+    /**
+     * @return Number of rows affected by the query.
+     */
+    public long rowsAffected() {
+        return rowsAffected;
+    }
+
+    /**
+     * @return Index of the set which caused an error or -1 if no error occurred.
+     */
+    public long errorSetIdx() {
+        return errorSetIdx;
+    }
+
+    /**
+     * @return Error message.
+     */
+    @Nullable public String errorMessage() {
+        return errorMessage;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
index c0d1c601c92b7..029135cd52e80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteRequest.java
@@ -42,10 +42,12 @@ public class OdbcQueryExecuteRequest extends OdbcRequest {
      * @param sqlQry SQL query.
      * @param args Arguments list.
      */
-    public OdbcQueryExecuteRequest(String cacheName, String sqlQry, Object[] args) {
+    public OdbcQueryExecuteRequest(@Nullable String cacheName, String sqlQry, Object[] args) {
         super(EXECUTE_SQL_QUERY);
 
-        this.cacheName = cacheName.isEmpty() ? null : cacheName;
+        assert sqlQry != null : "SQL query should not be null";
+
+        this.cacheName = cacheName;
         this.sqlQry = sqlQry;
         this.args = args;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java
index a4038123d2d4d..efa432b615614 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcQueryExecuteResult.java
@@ -41,7 +41,7 @@ public OdbcQueryExecuteResult(long queryId, Collection<OdbcColumnMeta> columnsMe
     /**
      * @return Query ID.
      */
-    public long getQueryId() {
+    public long queryId() {
         return queryId;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java
index f056fedf079bd..3c0c12e09cd28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequest.java
@@ -42,6 +42,9 @@ public class OdbcRequest {
     /** Get parameters meta. */
     public static final int GET_PARAMS_META = 7;
 
+    /** Execute sql query with the batch of parameters. */
+    public static final int QRY_EXEC_BATCH = 8;
+
     /** Command. */
     private final int cmd;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
index f922d9acc09be..ccced5eacac77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.internal.processors.odbc;
 
 import org.apache.ignite.IgniteCache;
+
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
@@ -36,7 +38,11 @@
 import java.sql.ParameterMetaData;
 import java.sql.PreparedStatement;
 import java.sql.Types;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -104,6 +110,9 @@ public OdbcResponse handle(long reqId, OdbcRequest req) {
                 case HANDSHAKE:
                     return performHandshake(reqId, (OdbcHandshakeRequest)req);
 
+                case QRY_EXEC_BATCH:
+                    return executeBatchQuery((OdbcQueryExecuteBatchRequest)req);
+
                 case EXECUTE_SQL_QUERY:
                     return executeQuery(reqId, (OdbcQueryExecuteRequest)req);
 
@@ -229,6 +238,101 @@ private OdbcResponse executeQuery(long reqId, OdbcQueryExecuteRequest req) {
         }
     }
 
+    /**
+     * {@link OdbcQueryExecuteBatchRequest} command handler.
+     *
+     * @param req Execute query request.
+     * @return Response.
+     */
+    private OdbcResponse executeBatchQuery(OdbcQueryExecuteBatchRequest req) {
+        long rowsAffected = 0;
+        int currentSet = 0;
+
+        try {
+            String sql = OdbcEscapeUtils.parse(req.sqlQuery());
+
+            if (log.isDebugEnabled())
+                log.debug("ODBC query parsed [original=" + req.sqlQuery() + ", parsed=" + sql + ']');
+
+            SqlFieldsQuery qry = new SqlFieldsQuery(sql);
+
+            qry.setDistributedJoins(distributedJoins);
+            qry.setEnforceJoinOrder(enforceJoinOrder);
+
+            IgniteCache<Object, Object> cache0 = ctx.grid().cache(req.schema());
+
+            if (cache0 == null)
+                return new OdbcResponse(OdbcResponse.STATUS_FAILED,
+                        "Cache doesn't exist (did you configure it?): " + req.schema());
+
+            IgniteCache<Object, Object> cache = cache0.withKeepBinary();
+
+            if (cache == null)
+                return new OdbcResponse(OdbcResponse.STATUS_FAILED,
+                        "Can not get cache with keep binary: " + req.schema());
+
+            Object[][] paramSet = req.arguments();
+
+            if (paramSet.length <= 0)
+                throw new IgniteException("Batch execute request with non-positive batch length. [len="
+                        + paramSet.length + ']');
+
+            // Getting meta and do the checks for the first execution.
+            qry.setArgs(paramSet[0]);
+
+            QueryCursorImpl<List<?>> qryCur = (QueryCursorImpl<List<?>>)cache.query(qry);
+
+            if (qryCur.isQuery())
+                throw new IgniteException("Batching of parameters only supported for DML statements. [query=" +
+                        req.sqlQuery() + ']');
+
+            rowsAffected += getRowsAffected(qryCur);
+
+            for (currentSet = 1; currentSet < paramSet.length; ++currentSet)
+            {
+                qry.setArgs(paramSet[currentSet]);
+
+                QueryCursor<List<?>> cur = cache.query(qry);
+
+                rowsAffected += getRowsAffected(cur);
+            }
+
+            OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected);
+
+            return new OdbcResponse(res);
+        }
+        catch (Exception e) {
+            U.error(log, "Failed to execute SQL query [req=" + req + ']', e);
+
+            OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected, currentSet,
+                    e.getMessage());
+
+            return new OdbcResponse(res);
+        }
+    }
+
+    /**
+     * Get affected rows for DML statement.
+     * @param qryCur Cursor.
+     * @return Number of table rows affected.
+     */
+    private static long getRowsAffected(QueryCursor<List<?>> qryCur) {
+        Iterator<List<?>> iter = qryCur.iterator();
+
+        if (iter.hasNext()) {
+            List<?> res = iter.next();
+
+            if (res.size() > 0) {
+                Long affected = (Long) res.get(0);
+
+                if (affected != null)
+                    return affected;
+            }
+        }
+
+        return 0;
+    }
+
     /**
      * {@link OdbcQueryCloseRequest} command handler.
      *
diff --git a/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp b/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp
index 82521be53d6e7..0149d6a801bfe 100644
--- a/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/application_data_buffer_test.cpp
@@ -44,9 +44,8 @@ BOOST_AUTO_TEST_CASE(TestPutIntToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     appBuf.PutInt8(12);
     BOOST_CHECK(!strcmp(buffer, "12"));
@@ -77,9 +76,8 @@ BOOST_AUTO_TEST_CASE(TestPutFloatToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     appBuf.PutFloat(12.42f);
     BOOST_CHECK(!strcmp(buffer, "12.42"));
@@ -102,9 +100,8 @@ BOOST_AUTO_TEST_CASE(TestPutGuidToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     ignite::Guid guid(0x1da1ef8f39ff4d62ULL, 0x8b72e8e9f3371801ULL);
 
@@ -119,7 +116,7 @@ BOOST_AUTO_TEST_CASE(TestGetGuidFromString)
     char buffer[] = "1da1ef8f-39ff-4d62-8b72-e8e9f3371801";
     SqlLen reslen = sizeof(buffer) - 1;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer) - 1, &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer) - 1, &reslen);
 
     ignite::Guid guid = appBuf.GetGuid();
 
@@ -130,9 +127,8 @@ BOOST_AUTO_TEST_CASE(TestPutBinaryToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     uint8_t binary[] = { 0x21, 0x84, 0xF4, 0xDC, 0x01, 0x00, 0xFF, 0xF0 };
 
@@ -146,9 +142,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToString)
 {
     char buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, buffer, sizeof(buffer), &reslen);
 
     std::string testString("Test string");
 
@@ -162,9 +157,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToWstring)
 {
     wchar_t buffer[1024];
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_WCHAR, buffer, sizeof(buffer), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_WCHAR, buffer, sizeof(buffer), &reslen);
 
     std::string testString("Test string");
 
@@ -176,9 +170,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToLong)
 {
     long numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutString("424242424");
     BOOST_CHECK(numBuf == 424242424L);
@@ -191,9 +184,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToTiny)
 {
     int8_t numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_TINYINT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_TINYINT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutString("12");
     BOOST_CHECK(numBuf == 12);
@@ -206,9 +198,8 @@ BOOST_AUTO_TEST_CASE(TestPutStringToFloat)
 {
     float numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutString("12.21");
     BOOST_CHECK_CLOSE_FRACTION(numBuf, 12.21, FLOAT_PRECISION);
@@ -221,9 +212,8 @@ BOOST_AUTO_TEST_CASE(TestPutIntToFloat)
 {
     float numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutInt8(5);
     BOOST_CHECK_CLOSE_FRACTION(numBuf, 5.0, FLOAT_PRECISION);
@@ -248,9 +238,8 @@ BOOST_AUTO_TEST_CASE(TestPutFloatToShort)
 {
     short numBuf;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &numBuf, sizeof(numBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &numBuf, sizeof(numBuf), &reslen);
 
     appBuf.PutDouble(5.42);
     BOOST_CHECK(numBuf == 5);
@@ -270,7 +259,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToDouble)
     double numBuf;
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, sizeof(numBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, sizeof(numBuf), &reslen);
 
     common::Decimal decimal;
 
@@ -299,7 +288,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToLong)
     long numBuf;
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, sizeof(numBuf), &reslen);
 
     common::Decimal decimal;
 
@@ -326,7 +315,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToString)
     char strBuf[64];
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     common::Decimal decimal;
 
@@ -353,7 +342,7 @@ BOOST_AUTO_TEST_CASE(TestPutDecimalToNumeric)
     SQL_NUMERIC_STRUCT buf;
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_NUMERIC, &buf, sizeof(buf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_NUMERIC, &buf, sizeof(buf), &reslen);
 
     common::Decimal decimal;
 
@@ -404,7 +393,7 @@ BOOST_AUTO_TEST_CASE(TestPutDateToString)
     char strBuf[64] = { 0 };
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     Date date = common::MakeDateGmt(1999, 2, 22);
 
@@ -418,7 +407,7 @@ BOOST_AUTO_TEST_CASE(TestPutTimestampToString)
     char strBuf[64] = { 0 };
     SqlLen reslen = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, 0);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     Timestamp date = common::MakeTimestampGmt(2018, 11, 1, 17, 45, 59);
 
@@ -432,10 +421,7 @@ BOOST_AUTO_TEST_CASE(TestPutDateToDate)
     SQL_DATE_STRUCT buf = { 0 };
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Date date = common::MakeDateGmt(1984, 5, 27);
 
@@ -451,10 +437,7 @@ BOOST_AUTO_TEST_CASE(TestPutTimestampToDate)
     SQL_DATE_STRUCT buf = { 0 };
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = common::MakeTimestampGmt(2004, 8, 14, 6, 34, 51, 573948623);
 
@@ -470,10 +453,7 @@ BOOST_AUTO_TEST_CASE(TestPutTimestampToTimestamp)
     SQL_TIMESTAMP_STRUCT buf = { 0 };
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = common::MakeTimestampGmt(2004, 8, 14, 6, 34, 51, 573948623);
 
@@ -494,10 +474,7 @@ BOOST_AUTO_TEST_CASE(TestPutDateToTimestamp)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Date date = common::MakeDateGmt(1984, 5, 27);
 
@@ -516,9 +493,8 @@ BOOST_AUTO_TEST_CASE(TestGetStringFromLong)
 {
     long numBuf = 42;
     SqlLen reslen = sizeof(numBuf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_SIGNED_LONG, &numBuf, reslen, &reslen);
 
     std::string res = appBuf.GetString(32);
 
@@ -535,9 +511,8 @@ BOOST_AUTO_TEST_CASE(TestGetStringFromDouble)
 {
     double numBuf = 43.36;
     SqlLen reslen = sizeof(numBuf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &numBuf, reslen, &reslen);
 
     std::string res = appBuf.GetString(32);
 
@@ -554,9 +529,8 @@ BOOST_AUTO_TEST_CASE(TestGetStringFromString)
 {
     char buf[] = "Some data 32d2d5hs";
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen);
 
     std::string res = appBuf.GetString(reslen);
 
@@ -567,9 +541,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromUshort)
 {
     unsigned short numBuf = 7162;
     SqlLen reslen = sizeof(numBuf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT, &numBuf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT, &numBuf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -584,9 +557,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromString)
 {
     char buf[] = "28.562";
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -601,9 +573,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromFloat)
 {
     float buf = 207.49f;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -618,9 +589,8 @@ BOOST_AUTO_TEST_CASE(TestGetFloatFromDouble)
 {
     double buf = 893.162;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen);
 
     float resFloat = appBuf.GetFloat();
 
@@ -635,9 +605,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromString)
 {
     char buf[] = "39";
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -660,9 +629,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromFloat)
 {
     float buf = -107.49f;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_FLOAT, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -685,9 +653,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromDouble)
 {
     double buf = 42.97f;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_DOUBLE, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -710,9 +677,8 @@ BOOST_AUTO_TEST_CASE(TestGetIntFromBigint)
 {
     uint64_t buf = 19;
     SqlLen reslen = sizeof(buf);
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf, reslen, &reslen, &offset);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf, reslen, &reslen);
 
     int64_t resInt64 = appBuf.GetInt64();
 
@@ -744,22 +710,19 @@ BOOST_AUTO_TEST_CASE(TestGetIntWithOffset)
         { 42, sizeof(uint64_t) }
     };
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen);
 
     int64_t val = appBuf.GetInt64();
 
     BOOST_CHECK(val == 12);
 
-    offset += sizeof(TestStruct);
+    appBuf.SetByteOffset(sizeof(TestStruct));
 
     val = appBuf.GetInt64();
 
     BOOST_CHECK(val == 42);
 
-    offsetPtr = 0;
+    appBuf.SetByteOffset(0);
 
     val = appBuf.GetInt64();
 
@@ -779,10 +742,7 @@ BOOST_AUTO_TEST_CASE(TestSetStringWithOffset)
         { "", 0 }
     };
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0].val, sizeof(buf[0].val), &buf[0].reslen);
 
     appBuf.PutString("Hello Ignite!");
 
@@ -792,7 +752,7 @@ BOOST_AUTO_TEST_CASE(TestSetStringWithOffset)
     BOOST_CHECK(res == "Hello Ignite!");
     BOOST_CHECK(res.size() == strlen("Hello Ignite!"));
 
-    offset += sizeof(TestStruct);
+    appBuf.SetByteOffset(sizeof(TestStruct));
 
     appBuf.PutString("Hello with offset!");
 
@@ -814,10 +774,7 @@ BOOST_AUTO_TEST_CASE(TestGetDateFromString)
     char buf[] = "1999-02-22";
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen);
 
     Date date = appBuf.GetDate();
 
@@ -840,10 +797,7 @@ BOOST_AUTO_TEST_CASE(TestGetTimestampFromString)
     char buf[] = "2018-11-01 17:45:59";
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_CHAR, &buf[0], sizeof(buf), &reslen);
 
     Timestamp date = appBuf.GetTimestamp();
 
@@ -871,10 +825,7 @@ BOOST_AUTO_TEST_CASE(TestGetDateFromDate)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Date date = appBuf.GetDate();
 
@@ -902,10 +853,7 @@ BOOST_AUTO_TEST_CASE(TestGetTimestampFromDate)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TDATE, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = appBuf.GetTimestamp();
 
@@ -937,10 +885,7 @@ BOOST_AUTO_TEST_CASE(TestGetTimestampFromTimestamp)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Timestamp ts = appBuf.GetTimestamp();
 
@@ -973,10 +918,7 @@ BOOST_AUTO_TEST_CASE(TestGetDateFromTimestamp)
 
     SqlLen reslen = sizeof(buf);
 
-    int offset = 0;
-    int* offsetPtr = &offset;
-
-    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen, &offsetPtr);
+    ApplicationDataBuffer appBuf(IGNITE_ODBC_C_TYPE_TTIMESTAMP, &buf, sizeof(buf), &reslen);
 
     Date date = appBuf.GetDate();
 
diff --git a/modules/platforms/cpp/odbc-test/src/column_test.cpp b/modules/platforms/cpp/odbc-test/src/column_test.cpp
index 66d0214d2cf60..a87ec094d84c2 100644
--- a/modules/platforms/cpp/odbc-test/src/column_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/column_test.cpp
@@ -66,9 +66,8 @@ BOOST_AUTO_TEST_CASE(TestColumnShort)
 
     short shortBuf = 0;
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &shortBuf, sizeof(shortBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_SHORT, &shortBuf, sizeof(shortBuf), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -114,9 +113,8 @@ BOOST_AUTO_TEST_CASE(TestColumnString)
 
     char strBuf[1024] = {};
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -164,9 +162,8 @@ BOOST_AUTO_TEST_CASE(TestColumnStringSeveral)
 
     std::string strBuf(data.size() / 3 + 2, 0);
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf[0], strBuf.size(), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf[0], strBuf.size(), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -246,9 +243,8 @@ BOOST_AUTO_TEST_CASE(TestColumnMultiString)
 
     char strBuf[1024] = {};
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
 
     BOOST_REQUIRE(column1.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -312,9 +308,8 @@ BOOST_AUTO_TEST_CASE(TestColumnByteArray)
 
     std::vector<int8_t> buf(data.size());
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_BINARY, &buf[0], buf.size(), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_BINARY, &buf[0], buf.size(), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -362,9 +357,8 @@ BOOST_AUTO_TEST_CASE(TestColumnByteArrayHalfBuffer)
 
     std::vector<int8_t> buf(data.size()/2);
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_BINARY, &buf[0], buf.size(), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_BINARY, &buf[0], buf.size(), &reslen);
 
     BOOST_REQUIRE(column.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
@@ -435,9 +429,8 @@ BOOST_AUTO_TEST_CASE(TestColumnByteArrayTwoColumns)
 
     int8_t buf[1024] = {};
     SqlLen reslen = 0;
-    int* offset = 0;
 
-    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_BINARY, &buf, sizeof(buf), &reslen, &offset);
+    ApplicationDataBuffer appBuf(type_traits::IGNITE_ODBC_C_TYPE_BINARY, &buf, sizeof(buf), &reslen);
 
     BOOST_REQUIRE(column1.ReadToBuffer(reader, appBuf) == SQL_RESULT_SUCCESS);
 
diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 422648e627246..8e528b4c0b910 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -33,6 +33,7 @@
 #include <boost/test/unit_test.hpp>
 
 #include "ignite/ignite.h"
+#include "ignite/common/fixed_size_array.h"
 #include "ignite/ignition.h"
 #include "ignite/impl/binary/binary_utils.h"
 
@@ -276,10 +277,10 @@ struct QueriesTestSuiteFixture
     }
 
     /**
-     * Insert requested number of TestType vlaues with all defaults except
+     * Insert requested number of TestType values with all defaults except
      * for the strFields, which are generated using getTestString().
      *
-     * @param num Number of records to insert.
+     * @param recordsNum Number of records to insert.
      * @param merge Set to true to use merge instead.
      */
     void InsertTestStrings(int recordsNum, bool merge = false)
@@ -337,6 +338,317 @@ struct QueriesTestSuiteFixture
             BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
     }
 
+    /**
+     * Insert requested number of TestType values in a batch.
+     *
+     * @param from Index to start from.
+     * @param to Index to stop.
+     * @param expectedToAffect Expected number of affected records.
+     * @param merge Set to true to use merge instead of insert.
+     * @return Records inserted.
+     */
+    int InsertTestBatch(int from, int to, int expectedToAffect, bool merge = false)
+    {
+        SQLCHAR insertReq[] = "INSERT "
+            "INTO TestType(_key, i8Field, i16Field, i32Field, strField, floatField, doubleField, boolField, dateField, "
+            "timestampField) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+        SQLCHAR mergeReq[] = "MERGE "
+            "INTO TestType(_key, i8Field, i16Field, i32Field, strField, floatField, doubleField, boolField, dateField, "
+            "timestampField) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+        SQLRETURN ret;
+
+        int recordsNum = to - from;
+
+        ret = SQLPrepare(stmt, merge ? mergeReq : insertReq, SQL_NTS);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        FixedSizeArray<int64_t> keys(recordsNum);
+        FixedSizeArray<int8_t> i8Fields(recordsNum);
+        FixedSizeArray<int16_t> i16Fields(recordsNum);
+        FixedSizeArray<int32_t> i32Fields(recordsNum);
+        FixedSizeArray<char> strFields(recordsNum * 1024);
+        FixedSizeArray<float> floatFields(recordsNum);
+        FixedSizeArray<double> doubleFields(recordsNum);
+        FixedSizeArray<bool> boolFields(recordsNum);
+        FixedSizeArray<SQL_DATE_STRUCT> dateFields(recordsNum);
+        FixedSizeArray<SQL_TIME_STRUCT> timeFields(recordsNum);
+        FixedSizeArray<SQL_TIMESTAMP_STRUCT> timestampFields(recordsNum);
+
+        FixedSizeArray<SQLLEN> strFieldsLen(recordsNum);
+
+        BOOST_CHECKPOINT("Filling param data");
+
+        for (int i = 0; i < recordsNum; ++i)
+        {
+            int seed = from + i;
+
+            keys[i] = seed;
+            i8Fields[i] = seed * 8;
+            i16Fields[i] = seed * 16;
+            i32Fields[i] = seed * 32;
+
+            std::string val = getTestString(seed);
+            strncpy(strFields.GetData() + 1024 * i, val.c_str(), 1023);
+            strFieldsLen[i] = val.size();
+
+            floatFields[i] = seed * 0.5f;
+            doubleFields[i] = seed * 0.25f;
+            boolFields[i] = seed % 2 == 0;
+
+            dateFields[i].year = 2017 + seed / 365;
+            dateFields[i].month = ((seed / 28) % 12) + 1;
+            dateFields[i].day = (seed % 28) + 1;
+
+            timeFields[i].hour = (seed / 3600) % 24;
+            timeFields[i].minute = (seed / 60) % 60;
+            timeFields[i].second = seed % 60;
+
+            timestampFields[i].year = dateFields[i].year;
+            timestampFields[i].month = dateFields[i].month;
+            timestampFields[i].day = dateFields[i].day;
+            timestampFields[i].hour = timeFields[i].hour;
+            timestampFields[i].minute = timeFields[i].minute;
+            timestampFields[i].second = timeFields[i].second;
+            timestampFields[i].fraction = std::abs(seed * 914873) % 1000000000;
+        }
+
+        SQLULEN setsProcessed = 0;
+
+        BOOST_CHECKPOINT("Setting processed pointer");
+        ret = SQLSetStmtAttr(stmt, SQL_ATTR_PARAMS_PROCESSED_PTR, &setsProcessed, SQL_IS_POINTER);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding keys");
+        ret = SQLBindParameter(stmt, 1, SQL_PARAM_INPUT, SQL_C_SBIGINT, SQL_BIGINT, 0, 0, keys.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding i8Fields");
+        ret = SQLBindParameter(stmt, 2, SQL_PARAM_INPUT, SQL_C_STINYINT, SQL_TINYINT, 0, 0, i8Fields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding i16Fields");
+        ret = SQLBindParameter(stmt, 3, SQL_PARAM_INPUT, SQL_C_SSHORT, SQL_SMALLINT, 0, 0, i16Fields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding i32Fields");
+        ret = SQLBindParameter(stmt, 4, SQL_PARAM_INPUT, SQL_C_SLONG, SQL_INTEGER, 0, 0, i32Fields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding strFields");
+        ret = SQLBindParameter(stmt, 5, SQL_PARAM_INPUT, SQL_C_CHAR, SQL_VARCHAR, 1024, 0, strFields.GetData(), 1024, strFieldsLen.GetData());
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding floatFields");
+        ret = SQLBindParameter(stmt, 6, SQL_PARAM_INPUT, SQL_C_FLOAT, SQL_FLOAT, 0, 0, floatFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding doubleFields");
+        ret = SQLBindParameter(stmt, 7, SQL_PARAM_INPUT, SQL_C_DOUBLE, SQL_DOUBLE, 0, 0, doubleFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding boolFields");
+        ret = SQLBindParameter(stmt, 8, SQL_PARAM_INPUT, SQL_C_BIT, SQL_BIT, 0, 0, boolFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding dateFields");
+        ret = SQLBindParameter(stmt, 9, SQL_PARAM_INPUT, SQL_C_DATE, SQL_DATE, 0, 0, dateFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Binding timestampFields");
+        ret = SQLBindParameter(stmt, 10, SQL_PARAM_INPUT, SQL_C_TIMESTAMP, SQL_TIMESTAMP, 0, 0, timestampFields.GetData(), 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Setting paramset size");
+        ret = SQLSetStmtAttr(stmt, SQL_ATTR_PARAMSET_SIZE, reinterpret_cast<SQLPOINTER>(recordsNum), 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Executing query");
+        ret = SQLExecute(stmt);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        SQLLEN affected = 0;
+        ret = SQLRowCount(stmt, &affected);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECK_EQUAL(affected, expectedToAffect);
+
+        BOOST_CHECKPOINT("Getting next result set");
+        ret = SQLMoreResults(stmt);
+
+        if (ret != SQL_NO_DATA)
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Resetting parameters.");
+        ret = SQLFreeStmt(stmt, SQL_RESET_PARAMS);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECKPOINT("Setting paramset size");
+        ret = SQLSetStmtAttr(stmt, SQL_ATTR_PARAMSET_SIZE, reinterpret_cast<SQLPOINTER>(1), 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        return static_cast<int>(setsProcessed);
+    }
+
+    void InsertBatchSelect(int recordsNum)
+    {
+        Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
+
+        // Inserting values.
+        int inserted = InsertTestBatch(0, recordsNum, recordsNum);
+
+        BOOST_REQUIRE_EQUAL(inserted, recordsNum);
+
+        int64_t key = 0;
+        char strField[1024] = { 0 };
+        SQLLEN strFieldLen = 0;
+
+        // Binding columns.
+        SQLRETURN ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Binding columns.
+        ret = SQLBindCol(stmt, 2, SQL_C_CHAR, &strField, sizeof(strField), &strFieldLen);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Just selecting everything to make sure everything is OK
+        SQLCHAR selectReq[] = "SELECT _key, strField FROM TestType ORDER BY _key";
+
+        ret = SQLExecDirect(stmt, selectReq, sizeof(selectReq));
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        int selectedRecordsNum = 0;
+
+        ret = SQL_SUCCESS;
+
+        while (ret == SQL_SUCCESS)
+        {
+            ret = SQLFetch(stmt);
+
+            if (ret == SQL_NO_DATA)
+                break;
+
+            if (!SQL_SUCCEEDED(ret))
+                BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+            std::string expectedStr = getTestString(selectedRecordsNum);
+            int64_t expectedKey = selectedRecordsNum;
+
+            BOOST_CHECK_EQUAL(key, expectedKey);
+
+            BOOST_CHECK_EQUAL(std::string(strField, strFieldLen), expectedStr);
+
+            ++selectedRecordsNum;
+        }
+
+        BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
+    }
+
+    void InsertNonFullBatchSelect(int recordsNum, int splitAt)
+    {
+        Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
+
+        // Inserting values.
+        int inserted = InsertTestBatch(splitAt, recordsNum, recordsNum - splitAt);
+
+        BOOST_REQUIRE_EQUAL(inserted, recordsNum - splitAt);
+
+        inserted = InsertTestBatch(0, recordsNum, splitAt);
+
+        BOOST_REQUIRE_EQUAL(inserted, splitAt);
+
+        int64_t key = 0;
+        char strField[1024] = { 0 };
+        SQLLEN strFieldLen = 0;
+
+        // Binding columns.
+        SQLRETURN ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Binding columns.
+        ret = SQLBindCol(stmt, 2, SQL_C_CHAR, &strField, sizeof(strField), &strFieldLen);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        // Just selecting everything to make sure everything is OK
+        SQLCHAR selectReq[] = "SELECT _key, strField FROM TestType ORDER BY _key";
+
+        ret = SQLExecDirect(stmt, selectReq, sizeof(selectReq));
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        int selectedRecordsNum = 0;
+
+        ret = SQL_SUCCESS;
+
+        while (ret == SQL_SUCCESS)
+        {
+            ret = SQLFetch(stmt);
+
+            if (ret == SQL_NO_DATA)
+                break;
+
+            if (!SQL_SUCCEEDED(ret))
+                BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+            std::string expectedStr = getTestString(selectedRecordsNum);
+            int64_t expectedKey = selectedRecordsNum;
+
+            BOOST_CHECK_EQUAL(key, expectedKey);
+
+            BOOST_CHECK_EQUAL(std::string(strField, strFieldLen), expectedStr);
+
+            ++selectedRecordsNum;
+        }
+
+        BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
+    }
+
     /** Node started during the test. */
     Ignite grid;
 
@@ -1313,6 +1625,76 @@ BOOST_AUTO_TEST_CASE(TestInsertMergeSelect)
     BOOST_CHECK_EQUAL(recordsNum, selectedRecordsNum);
 }
 
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2)
+{
+    InsertBatchSelect(2);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect100)
+{
+    InsertBatchSelect(100);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1000)
+{
+    InsertBatchSelect(1000);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1023)
+{
+    InsertBatchSelect(1024);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1024)
+{
+    InsertBatchSelect(1024);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect1025)
+{
+    InsertBatchSelect(1025);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2000)
+{
+    InsertBatchSelect(2000);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2047)
+{
+    InsertBatchSelect(2048);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2048)
+{
+    InsertBatchSelect(2048);
+}
+
+BOOST_AUTO_TEST_CASE(TestInsertBatchSelect2049)
+{
+    InsertBatchSelect(2049);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect900)
+{
+    InsertNonFullBatchSelect(900, 42);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect1500)
+{
+    InsertNonFullBatchSelect(1500, 100);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect4500)
+{
+    InsertNonFullBatchSelect(4500, 1500);
+}
+
+BOOST_AUTO_TEST_CASE(TestNotFullInsertBatchSelect4096)
+{
+    InsertNonFullBatchSelect(4096, 1024);
+}
+
 template<size_t n, size_t k>
 void CheckMeta(char columns[n][k], SQLLEN columnsLen[n])
 {
diff --git a/modules/platforms/cpp/odbc-test/src/row_test.cpp b/modules/platforms/cpp/odbc-test/src/row_test.cpp
index f38e9c5b58f1d..1ae34c12feadf 100644
--- a/modules/platforms/cpp/odbc-test/src/row_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/row_test.cpp
@@ -82,12 +82,11 @@ void CheckRowData(Row& row, size_t rowIdx)
     char strBuf[1024];
     SQLGUID guidBuf;
     char bitBuf;
-    int* offset = 0;
 
-    ApplicationDataBuffer appLongBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_LONG, &longBuf, sizeof(longBuf), &reslen, &offset);
-    ApplicationDataBuffer appStrBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen, &offset);
-    ApplicationDataBuffer appGuidBuf(type_traits::IGNITE_ODBC_C_TYPE_GUID, &guidBuf, sizeof(guidBuf), &reslen, &offset);
-    ApplicationDataBuffer appBitBuf(type_traits::IGNITE_ODBC_C_TYPE_BIT, &bitBuf, sizeof(bitBuf), &reslen, &offset);
+    ApplicationDataBuffer appLongBuf(type_traits::IGNITE_ODBC_C_TYPE_SIGNED_LONG, &longBuf, sizeof(longBuf), &reslen);
+    ApplicationDataBuffer appStrBuf(type_traits::IGNITE_ODBC_C_TYPE_CHAR, &strBuf, sizeof(strBuf), &reslen);
+    ApplicationDataBuffer appGuidBuf(type_traits::IGNITE_ODBC_C_TYPE_GUID, &guidBuf, sizeof(guidBuf), &reslen);
+    ApplicationDataBuffer appBitBuf(type_traits::IGNITE_ODBC_C_TYPE_BIT, &bitBuf, sizeof(bitBuf), &reslen);
 
     // Checking size.
     BOOST_REQUIRE(row.GetSize() == 4);
diff --git a/modules/platforms/cpp/odbc/Makefile.am b/modules/platforms/cpp/odbc/Makefile.am
index de6f75acf7bd6..8d7693c7a5d1a 100644
--- a/modules/platforms/cpp/odbc/Makefile.am
+++ b/modules/platforms/cpp/odbc/Makefile.am
@@ -52,6 +52,7 @@ libignite_odbc_la_SOURCES = \
     os/linux/src/system/socket_client.cpp \
     src/app/application_data_buffer.cpp \
     src/app/parameter.cpp \
+    src/app/parameter_set.cpp \
     src/common_types.cpp \
     src/config/configuration.cpp \
     src/config/connection_info.cpp \
@@ -68,6 +69,7 @@ libignite_odbc_la_SOURCES = \
     src/dsn_config.cpp \
     src/query/column_metadata_query.cpp \
     src/query/data_query.cpp \
+    src/query/batch_query.cpp \
     src/query/foreign_keys_query.cpp \
     src/query/primary_keys_query.cpp \
     src/query/table_metadata_query.cpp \
@@ -76,6 +78,7 @@ libignite_odbc_la_SOURCES = \
     src/protocol_version.cpp \
     src/result_page.cpp \
     src/row.cpp \
+    src/message.cpp \
     src/column.cpp \
     src/statement.cpp \
     src/type_traits.cpp \
diff --git a/modules/platforms/cpp/odbc/include/Makefile.am b/modules/platforms/cpp/odbc/include/Makefile.am
index 36d8e2535ed09..92c85c56715e1 100644
--- a/modules/platforms/cpp/odbc/include/Makefile.am
+++ b/modules/platforms/cpp/odbc/include/Makefile.am
@@ -22,6 +22,7 @@ noinst_HEADERS = \
     ignite/odbc/query/table_metadata_query.h \
     ignite/odbc/query/special_columns_query.h \
     ignite/odbc/query/type_info_query.h \
+    ignite/odbc/query/batch_query.h \
     ignite/odbc/query/data_query.h \
     ignite/odbc/query/foreign_keys_query.h \
     ignite/odbc/query/column_metadata_query.h \
@@ -35,6 +36,7 @@ noinst_HEADERS = \
     ignite/odbc/parser.h \
     ignite/odbc/app/application_data_buffer.h \
     ignite/odbc/app/parameter.h \
+    ignite/odbc/app/parameter_set.h \
     ignite/odbc/row.h \
     ignite/odbc/utility.h \
     ignite/odbc/environment.h \
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h b/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h
index 18ac36aead536..b7989c55982fa 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/app/application_data_buffer.h
@@ -54,10 +54,9 @@ namespace ignite
                  * @param buffer Data buffer pointer.
                  * @param buflen Data buffer length.
                  * @param reslen Resulting data length.
-                 * @param offset Pointer to buffer and reslen offset pointer.
                  */
                 ApplicationDataBuffer(type_traits::IgniteSqlType type, void* buffer,
-                    SqlLen buflen, SqlLen* reslen, int** offset = 0);
+                    SqlLen buflen, SqlLen* reslen);
 
                 /**
                  * Copy constructor.
@@ -80,13 +79,23 @@ namespace ignite
                 ApplicationDataBuffer& operator=(const ApplicationDataBuffer& other);
 
                 /**
-                 * Set pointer to offset pointer.
+                 * Set offset in bytes for all bound pointers.
                  *
-                 * @param offset Pointer to offset pointer.
+                 * @param offset Offset.
                  */
-                void SetPtrToOffsetPtr(int** offset)
+                void SetByteOffset(int offset)
                 {
-                    this->offset = offset;
+                    this->byteOffset = offset;
+                }
+
+                /**
+                 * Set offset in elements for all bound pointers.
+                 *
+                 * @param
+                 */
+                void SetElementOffset(SqlUlen idx)
+                {
+                    this->elementOffset = idx;
                 }
 
                 /**
@@ -313,6 +322,13 @@ namespace ignite
                  */
                 SqlLen GetDataAtExecSize() const;
 
+                /**
+                 * Get single element size.
+                 *
+                 * @return Size of the single element.
+                 */
+                SqlLen GetElementSize() const;
+
                 /**
                  * Get size of the input buffer.
                  *
@@ -392,10 +408,11 @@ namespace ignite
                  * Apply buffer offset to pointer.
                  * Adds offset to pointer if offset pointer is not null.
                  * @param ptr Pointer.
+                 * @param elemSize Element size.
                  * @return Pointer with applied offset.
                  */
                 template<typename T>
-                T* ApplyOffset(T* ptr) const;
+                T* ApplyOffset(T* ptr, size_t elemSize) const;
 
                 /** Underlying data type. */
                 type_traits::IgniteSqlType type;
@@ -409,8 +426,11 @@ namespace ignite
                 /** Result length. */
                 SqlLen* reslen;
 
-                /** Pointer to implementation pointer to application offset */
-                int** offset;
+                /** Current byte offset */
+                int byteOffset;
+
+                /** Current element offset. */
+                SqlUlen elementOffset;
             };
 
             /** Column binging map type alias. */
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h
index 1cf85b514aa59..cdaaead40ee3d 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter.h
@@ -22,12 +22,10 @@
 
 #include <map>
 
-#include <ignite/guid.h>
 #include <ignite/impl/binary/binary_writer_impl.h>
 #include <ignite/impl/binary/binary_reader_impl.h>
 
 #include "ignite/odbc/app/application_data_buffer.h"
-#include "ignite/odbc/type_traits.h"
 
 namespace ignite
 {
@@ -70,7 +68,7 @@ namespace ignite
                 ~Parameter();
 
                 /**
-                 * Copy assigment operator.
+                 * Assignment operator.
                  *
                  * @param other Other instance.
                  * @return This.
@@ -78,10 +76,12 @@ namespace ignite
                 Parameter& operator=(const Parameter& other);
 
                 /**
-                 * Write request using provided writer.
+                 * Write parameter using provided writer.
                  * @param writer Writer.
+                 * @param offset Offset for the buffer.
+                 * @param idx Index for the array-of-parameters case.
                  */
-                void Write(impl::binary::BinaryWriterImpl& writer) const;
+                void Write(impl::binary::BinaryWriterImpl& writer, int offset = 0, SqlUlen idx = 0) const;
 
                 /**
                  * Get data buffer.
@@ -90,6 +90,13 @@ namespace ignite
                  */
                 ApplicationDataBuffer& GetBuffer();
 
+                /**
+                 * Get data buffer.
+                 *
+                 * @return underlying ApplicationDataBuffer instance.
+                 */
+                const ApplicationDataBuffer& GetBuffer() const;
+
                 /**
                  * Reset stored at-execution data.
                  */
@@ -128,9 +135,6 @@ namespace ignite
                 /** Stored at-execution data. */
                 std::vector<int8_t> storedData;
             };
-
-            /** Parameter binging map type alias. */
-            typedef std::map<uint16_t, Parameter> ParameterBindingMap;
         }
     }
 }
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
new file mode 100644
index 0000000000000..2ab55808c073c
--- /dev/null
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _IGNITE_ODBC_APP_PARAMETER_SET
+#define _IGNITE_ODBC_APP_PARAMETER_SET
+
+#include <stdint.h>
+
+#include <map>
+
+#include <ignite/impl/binary/binary_writer_impl.h>
+#include <ignite/impl/binary/binary_reader_impl.h>
+
+#include "ignite/odbc/app/parameter.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        namespace app
+        {
+            /**
+             * Parameter set.
+             */
+            class ParameterSet
+            {
+                /** Parameter binging map type alias. */
+                typedef std::map<uint16_t, Parameter> ParameterBindingMap;
+
+                /** Parameter meta vector. */
+                typedef std::vector<int8_t> ParameterTypeVector;
+            public:
+                /**
+                 * Default constructor.
+                 */
+                ParameterSet();
+
+                /**
+                 * Destructor.
+                 */
+                ~ParameterSet()
+                {
+                    // No-op.
+                }
+
+                /**
+                 * Set parameters set size.
+                 *
+                 * @param size Size of the parameter set.
+                 */
+                void SetParamSetSize(SqlUlen size);
+
+                /**
+                 * Bind parameter.
+                 * 
+                 * @param paramIdx Parameter index.
+                 * @param param Parameter.
+                 */
+                void BindParameter(uint16_t paramIdx, const Parameter& param);
+
+                /**
+                 * Unbind specified parameter.
+                 *
+                 * @param paramIdx Parameter index.
+                 */
+                void UnbindParameter(uint16_t paramIdx);
+
+                /**
+                 * Unbind all parameters.
+                 */
+                void UnbindAll();
+
+                /**
+                 * Get number of binded parameters.
+                 *
+                 * @return Number of binded parameters.
+                 */
+                uint16_t GetParametersNumber() const;
+
+                /**
+                 * Set parameter binding offset pointer.
+                 *
+                 * @param ptr Parameter binding offset pointer.
+                 */
+                void SetParamBindOffsetPtr(int* ptr);
+
+                /**
+                 * Get parameter binding offset pointer.
+                 *
+                 * @return Parameter binding offset pointer.
+                 */
+                int* GetParamBindOffsetPtr();
+
+                /**
+                 * Prepare parameters set for statement execution.
+                 */
+                void Prepare();
+
+                /**
+                 * Check if the data at-execution is needed.
+                 *
+                 * @return True if the data at execution is needed.
+                 */
+                bool IsDataAtExecNeeded() const;
+
+                /**
+                 * Update parameter types metadata.
+                 *
+                 * @param meta Types metadata.
+                 */
+                void UpdateParamsTypes(const ParameterTypeVector& meta);
+
+                /**
+                 * Get type id of the parameter.
+                 *
+                 * @param idx Parameter index.
+                 * @param dflt Default value to return if the type can not be found.
+                 * @return Type ID of the parameter or dflt, if the type can not be returned.
+                 */
+                int8_t GetParamType(int16_t idx, int8_t dflt);
+
+                /**
+                 * Get expected parameters number.
+                 * Using metadata. If metadata was not updated returns zero.
+                 *
+                 * @return Expected parameters number.
+                 */
+                uint16_t GetExpectedParamNum();
+
+                /**
+                 * Check if the metadata was set for the parameter set.
+                 * 
+                 * @return True if the metadata was set for the parameter set. 
+                 */
+                bool IsMetadataSet() const;
+
+                /**
+                 * Check if the parameter selected for putting data at-execution.
+                 *
+                 * @return True if the parameter selected for putting data at-execution.
+                 */
+                bool IsParameterSelected() const;
+
+                /**
+                 * Get parameter by index.
+                 *
+                 * @param idx Index.
+                 * @return Parameter or null, if parameter is not bound.
+                 */
+                Parameter* GetParameter(uint16_t idx);
+
+                /**
+                 * Get selected parameter.
+                 *
+                 * @return Parameter or null, if parameter is not bound.
+                 */
+                Parameter* GetSelectedParameter();
+
+                /**
+                 * Internally selects next parameter for putting data at-execution.
+                 *
+                 * @return Parameter if found and null otherwise.
+                 */
+                Parameter* SelectNextParameter();
+
+                /**
+                 * Write only first row of the param set using provided writer.
+                 * @param writer Writer.
+                 */
+                void Write(impl::binary::BinaryWriterImpl& writer) const;
+
+                /**
+                 * Write rows of the param set in interval [begin, end) using provided writer.
+                 * @param writer Writer.
+                 * @param begin Beginng of the interval.
+                 * @param end End of the interval.
+                 * @param last Last page flag.
+                 */
+                void Write(impl::binary::BinaryWriterImpl& writer, SqlUlen begin, SqlUlen end, bool last) const;
+
+                /**
+                 * Calculate row length.
+                 *
+                 * @return Row length.
+                 */
+                int32_t CalculateRowLen() const;
+
+                /**
+                 * Get parameter set size.
+                 *
+                 * @return Number of rows in set.
+                 */
+                int32_t GetParamSetSize() const;
+
+                /**
+                 * Set number of parameters processed in batch.
+                 *
+                 * @param processed Processed.
+                 */
+                void SetParamsProcessed(SqlUlen processed) const;
+
+                /**
+                 * Number of processed params should be written using provided address.
+                 *
+                 * @param ptr Pointer.
+                 */
+                void SetParamsProcessedPtr(SqlUlen* ptr);
+
+                /**
+                 * Get pointer to write number of parameters processed in batch.
+                 *
+                 * @return Pointer to write number of parameters processed in batch.
+                 */
+                SqlUlen* GetParamsProcessedPtr();
+
+            private:
+                /**
+                 * Write single row of the param set using provided writer.
+                 * @param writer Writer.
+                 * @param idx Row index.
+                 */
+                void WriteRow(impl::binary::BinaryWriterImpl& writer, SqlUlen idx) const;
+
+                IGNITE_NO_COPY_ASSIGNMENT(ParameterSet);
+
+                /** Parameters. */
+                ParameterBindingMap parameters;
+
+                /** Parameter types. */
+                ParameterTypeVector paramTypes;
+
+                /** Offset added to pointers to change binding of parameters. */
+                int* paramBindOffset;
+
+                /** Processed parameters. */
+                SqlUlen* processedParamRows;
+
+                /** Parameter set size. */
+                SqlUlen paramSetSize;
+
+                /** Current position in parametor set. */
+                SqlUlen paramSetPos;
+
+                /** Index of the parameter, which is currently being set. */
+                uint16_t currentParamIdx;
+
+                /** Parameter types are set. */
+                bool typesSet;
+            };
+        }
+    }
+}
+
+#endif //_IGNITE_ODBC_APP_PARAMETER_SET
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h b/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
index 5d8901d6c2ef0..fc25b56822c66 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
@@ -156,9 +156,7 @@ namespace ignite
              */
             SQL_STATE_HYT01_CONNECTIOIN_TIMEOUT,
 
-            /**
-             * Driver does not support this function.
-             */
+            /** Driver does not support this function. */
             SQL_STATE_IM001_FUNCTION_NOT_SUPPORTED
         };
 
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h b/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
index 6937fcc6f3196..95e4f548c1f9c 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
@@ -45,14 +45,14 @@ namespace ignite
                  *
                  * @return Diagnostic record.
                  */
-                virtual const diagnostic::DiagnosticRecordStorage& GetDiagnosticRecords() const = 0;
+                virtual const DiagnosticRecordStorage& GetDiagnosticRecords() const = 0;
 
                 /**
                  * Get diagnostic record.
                  *
                  * @return Diagnostic record.
                  */
-                virtual diagnostic::DiagnosticRecordStorage& GetDiagnosticRecords() = 0;
+                virtual DiagnosticRecordStorage& GetDiagnosticRecords() = 0;
 
                 /**
                  * Add new status record.
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
index a2bbd99adce3c..cecdac67f9d36 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
@@ -24,11 +24,11 @@
 #include "ignite/impl/binary/binary_writer_impl.h"
 #include "ignite/impl/binary/binary_reader_impl.h"
 
-#include "ignite/odbc/utility.h"
 #include "ignite/odbc/result_page.h"
+#include "ignite/odbc/protocol_version.h"
 #include "ignite/odbc/meta/column_meta.h"
 #include "ignite/odbc/meta/table_meta.h"
-#include "ignite/odbc/app/parameter.h"
+#include "ignite/odbc/app/parameter_set.h"
 
 namespace ignite
 {
@@ -48,7 +48,9 @@ namespace ignite
 
             REQUEST_TYPE_GET_TABLES_METADATA = 6,
 
-            REQUEST_TYPE_GET_PARAMS_METADATA = 7
+            REQUEST_TYPE_GET_PARAMS_METADATA = 7,
+
+            REQUEST_TYPE_EXECUTE_SQL_QUERY_BATCH = 8
         };
 
         enum ResponseStatus
@@ -71,35 +73,18 @@ namespace ignite
              * @param distributedJoins Distributed joins flag.
              * @param enforceJoinOrder Enforce join order flag.
              */
-            HandshakeRequest(int64_t version, bool distributedJoins, bool enforceJoinOrder) :
-                version(version),
-                distributedJoins(distributedJoins),
-                enforceJoinOrder(enforceJoinOrder)
-            {
-                // No-op.
-            }
+            HandshakeRequest(int64_t version, bool distributedJoins, bool enforceJoinOrder);
 
             /**
              * Destructor.
              */
-            ~HandshakeRequest()
-            {
-                // No-op.
-            }
+            ~HandshakeRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_HANDSHAKE);
-
-                writer.WriteInt64(version);
-
-                writer.WriteBool(distributedJoins);
-                writer.WriteBool(enforceJoinOrder);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Protocol version. */
@@ -123,65 +108,80 @@ namespace ignite
              *
              * @param cache Cache name.
              * @param sql SQL query.
-             * @param argsNum Number of arguments.
+             * @param params Number of arguments.
              */
-            QueryExecuteRequest(const std::string& cache, const std::string& sql,
-                const app::ParameterBindingMap& params) :
-                cache(cache),
-                sql(sql),
-                params(params)
-            {
-                // No-op.
-            }
+            QueryExecuteRequest(const std::string& cache, const std::string& sql, const app::ParameterSet& params);
 
             /**
              * Destructor.
              */
-            ~QueryExecuteRequest()
-            {
-                // No-op.
-            }
+            ~QueryExecuteRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_EXECUTE_SQL_QUERY);
-                utility::WriteString(writer, cache);
-                utility::WriteString(writer, sql);
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
-                writer.WriteInt32(static_cast<int32_t>(params.size()));
+        private:
+            /** Cache name. */
+            std::string cache;
 
-                app::ParameterBindingMap::const_iterator i;
-                uint16_t prev = 0;
+            /** SQL query. */
+            std::string sql;
 
-                for (i = params.begin(); i != params.end(); ++i) {
-                    uint16_t current = i->first;
+            /** Parameters bindings. */
+            const app::ParameterSet& params;
+        };
 
-                    while ((current - prev) > 1) {
-                        writer.WriteNull();
-                        ++prev;
-                    }
+        /**
+         * Query execute batch request.
+         */
+        class QueryExecuteBatchtRequest
+        {
+        public:
+            /**
+             * Constructor.
+             *
+             * @param schema Schema.
+             * @param sql SQL query.
+             * @param params Query arguments.
+             * @param begin Beginng of the interval.
+             * @param end End of the interval.
+             */
+            QueryExecuteBatchtRequest(const std::string& schema, const std::string& sql,
+                                          const app::ParameterSet& params, SqlUlen begin, SqlUlen end, bool last);
 
-                    i->second.Write(writer);
+            /**
+             * Destructor.
+             */
+            ~QueryExecuteBatchtRequest();
 
-                    prev = current;
-                }
-            }
+            /**
+             * Write request using provided writer.
+             * @param writer Writer.
+             */
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
-            /** Cache name. */
-            std::string cache;
+            /** Schema name. */
+            std::string schema;
 
             /** SQL query. */
             std::string sql;
 
             /** Parameters bindings. */
-            const app::ParameterBindingMap& params;
-        };
+            const app::ParameterSet& params;
 
+            /** Beginng of the interval. */
+            SqlUlen begin;
+
+            /** End of the interval. */
+            SqlUlen end;
+
+            /** Last page flag. */
+            bool last;
+        };
 
         /**
          * Query close request.
@@ -194,28 +194,18 @@ namespace ignite
              *
              * @param queryId Query ID.
              */
-            QueryCloseRequest(int64_t queryId) : queryId(queryId)
-            {
-                // No-op.
-            }
+            QueryCloseRequest(int64_t queryId);
 
             /**
              * Destructor.
              */
-            ~QueryCloseRequest()
-            {
-                // No-op.
-            }
+            ~QueryCloseRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_CLOSE_SQL_QUERY);
-                writer.WriteInt64(queryId);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Query ID. */
@@ -234,31 +224,18 @@ namespace ignite
              * @param queryId Query ID.
              * @param pageSize Required page size.
              */
-            QueryFetchRequest(int64_t queryId, int32_t pageSize) :
-                queryId(queryId),
-                pageSize(pageSize)
-            {
-                // No-op.
-            }
+            QueryFetchRequest(int64_t queryId, int32_t pageSize);
 
             /**
              * Destructor.
              */
-            ~QueryFetchRequest()
-            {
-                // No-op.
-            }
+            ~QueryFetchRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_FETCH_SQL_QUERY);
-                writer.WriteInt64(queryId);
-                writer.WriteInt32(pageSize);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Query ID. */
@@ -281,34 +258,18 @@ namespace ignite
              * @param table Table name.
              * @param column Column name.
              */
-            QueryGetColumnsMetaRequest(const std::string& schema, const std::string& table, const std::string& column) :
-                schema(schema),
-                table(table),
-                column(column)
-            {
-                // No-op.
-            }
+            QueryGetColumnsMetaRequest(const std::string& schema, const std::string& table, const std::string& column);
 
             /**
              * Destructor.
              */
-            ~QueryGetColumnsMetaRequest()
-            {
-                // No-op.
-            }
+            ~QueryGetColumnsMetaRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_GET_COLUMNS_METADATA);
-                
-                utility::WriteString(writer, schema);
-                utility::WriteString(writer, table);
-                utility::WriteString(writer, column);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Schema search pattern. */
@@ -336,36 +297,18 @@ namespace ignite
              * @param tableTypes Table types search pattern.
              */
             QueryGetTablesMetaRequest(const std::string& catalog, const std::string& schema,
-                                      const std::string& table, const std::string& tableTypes) :
-                catalog(catalog),
-                schema(schema),
-                table(table),
-                tableTypes(tableTypes)
-            {
-                // No-op.
-            }
+                                      const std::string& table, const std::string& tableTypes);
 
             /**
              * Destructor.
              */
-            ~QueryGetTablesMetaRequest()
-            {
-                // No-op.
-            }
+            ~QueryGetTablesMetaRequest();
 
             /**
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_GET_TABLES_METADATA);
-
-                utility::WriteString(writer, catalog);
-                utility::WriteString(writer, schema);
-                utility::WriteString(writer, table);
-                utility::WriteString(writer, tableTypes);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Column search pattern. */
@@ -414,13 +357,7 @@ namespace ignite
              * Write request using provided writer.
              * @param writer Writer.
              */
-            void Write(ignite::impl::binary::BinaryWriterImpl& writer) const
-            {
-                writer.WriteInt8(REQUEST_TYPE_GET_PARAMS_METADATA);
-
-                utility::WriteString(writer, cacheName);
-                utility::WriteString(writer, sqlQuery);
-            }
+            void Write(impl::binary::BinaryWriterImpl& writer) const;
 
         private:
             /** Cache name. */
@@ -439,33 +376,19 @@ namespace ignite
             /**
              * Constructor.
              */
-            Response() : status(RESPONSE_STATUS_FAILED), error()
-            {
-                // No-op.
-            }
+            Response();
 
             /**
              * Destructor.
              */
-            virtual ~Response()
-            {
-                // No-op.
-            }
+            virtual ~Response();
 
             /**
              * Read response using provided reader.
              * @param reader Reader.
              */
-            void Read(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                status = reader.ReadInt8();
+            void Read(impl::binary::BinaryReaderImpl& reader);
 
-                if (status == RESPONSE_STATUS_SUCCESS)
-                    ReadOnSuccess(reader);
-                else
-                    utility::ReadString(reader, error);;
-            }
-            
             /**
              * Get request processing status.
              * @return Status.
@@ -488,10 +411,7 @@ namespace ignite
             /**
              * Read data if response status is RESPONSE_STATUS_SUCCESS.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl&)
-            {
-                // No-op.
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl&);
 
         private:
             /** Request processing status. */
@@ -510,21 +430,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            HandshakeResponse() :
-                accepted(false),
-                protoVerSince(),
-                currentVer()
-            {
-                // No-op.
-            }
+            HandshakeResponse();
 
             /**
              * Destructor.
              */
-            ~HandshakeResponse()
-            {
-                // No-op.
-            }
+            ~HandshakeResponse();
 
             /**
              * Check if the handshake has been accepted.
@@ -558,16 +469,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                accepted = reader.ReadBool();
-
-                if (!accepted)
-                {
-                    utility::ReadString(reader, protoVerSince);
-                    utility::ReadString(reader, currentVer);
-                }
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Handshake accepted. */
             bool accepted;
@@ -588,18 +490,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryCloseResponse() : queryId(0)
-            {
-                // No-op.
-            }
+            QueryCloseResponse();
 
             /**
              * Destructor.
              */
-            ~QueryCloseResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryCloseResponse();
 
             /**
              * Get query ID.
@@ -615,10 +511,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                queryId = reader.ReadInt64();
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Query ID. */
             int64_t queryId;
@@ -633,18 +526,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryExecuteResponse() : queryId(0), meta()
-            {
-                // No-op.
-            }
+            QueryExecuteResponse();
 
             /**
              * Destructor.
              */
-            ~QueryExecuteResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryExecuteResponse();
 
             /**
              * Get query ID.
@@ -669,12 +556,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                queryId = reader.ReadInt64();
-
-                meta::ReadColumnMetaVector(reader, meta);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Query ID. */
             int64_t queryId;
@@ -684,28 +566,82 @@ namespace ignite
         };
 
         /**
-         * Query fetch response.
+         * Query execute batch start response.
          */
-        class QueryFetchResponse : public Response
+        class QueryExecuteBatchResponse : public Response
         {
         public:
             /**
              * Constructor.
-             * @param resultPage Result page.
              */
-            QueryFetchResponse(ResultPage& resultPage) : queryId(0), resultPage(resultPage)
+            QueryExecuteBatchResponse();
+
+            /**
+             * Destructor.
+             */
+            virtual ~QueryExecuteBatchResponse();
+
+            /**
+             * Affected rows.
+             * @return Affected rows.
+             */
+            int64_t GetAffectedRows() const
             {
-                // No-op.
+                return affectedRows;
             }
 
             /**
-             * Destructor.
+             * Get index of the set which caused an error.
+             * @return Index of the set which caused an error.
              */
-            ~QueryFetchResponse()
+            int64_t GetErrorSetIdx() const
             {
-                // No-op.
+                return affectedRows;
+            }
+
+            /**
+             * Get error message.
+             * @return Error message.
+             */
+            const std::string& GetErrorMessage() const
+            {
+                return errorMessage;
             }
 
+        private:
+            /**
+             * Read response using provided reader.
+             * @param reader Reader.
+             */
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
+
+            /** Affected rows. */
+            int64_t affectedRows;
+
+            /** Index of the set which caused an error. */
+            int64_t errorSetIdx;
+
+            /** Error message. */
+            std::string errorMessage;
+        };
+
+        /**
+         * Query fetch response.
+         */
+        class QueryFetchResponse : public Response
+        {
+        public:
+            /**
+             * Constructor.
+             * @param resultPage Result page.
+             */
+            QueryFetchResponse(ResultPage& resultPage);
+
+            /**
+             * Destructor.
+             */
+            virtual ~QueryFetchResponse();
+
             /**
              * Get query ID.
              * @return Query ID.
@@ -720,12 +656,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                queryId = reader.ReadInt64();
-
-                resultPage.Read(reader);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Query ID. */
             int64_t queryId;
@@ -743,18 +674,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryGetColumnsMetaResponse()
-            {
-                // No-op.
-            }
+            QueryGetColumnsMetaResponse();
 
             /**
              * Destructor.
              */
-            ~QueryGetColumnsMetaResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryGetColumnsMetaResponse();
 
             /**
              * Get column metadata.
@@ -770,10 +695,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                meta::ReadColumnMetaVector(reader, meta);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Columns metadata. */
             meta::ColumnMetaVector meta;
@@ -788,18 +710,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryGetTablesMetaResponse()
-            {
-                // No-op.
-            }
+            QueryGetTablesMetaResponse();
 
             /**
              * Destructor.
              */
-            ~QueryGetTablesMetaResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryGetTablesMetaResponse();
 
             /**
              * Get column metadata.
@@ -815,10 +731,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                meta::ReadTableMetaVector(reader, meta);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Columns metadata. */
             meta::TableMetaVector meta;
@@ -833,18 +746,12 @@ namespace ignite
             /**
              * Constructor.
              */
-            QueryGetParamsMetaResponse()
-            {
-                // No-op.
-            }
+            QueryGetParamsMetaResponse();
 
             /**
              * Destructor.
              */
-            ~QueryGetParamsMetaResponse()
-            {
-                // No-op.
-            }
+            virtual ~QueryGetParamsMetaResponse();
 
             /**
              * Get parameter type IDs.
@@ -860,10 +767,7 @@ namespace ignite
              * Read response using provided reader.
              * @param reader Reader.
              */
-            virtual void ReadOnSuccess(ignite::impl::binary::BinaryReaderImpl& reader)
-            {
-                utility::ReadByteArray(reader, typeIds);
-            }
+            virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
 
             /** Columns metadata. */
             std::vector<int8_t> typeIds;
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
new file mode 100644
index 0000000000000..a691f7351d956
--- /dev/null
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _IGNITE_ODBC_QUERY_BATCH_QUERY
+#define _IGNITE_ODBC_QUERY_BATCH_QUERY
+
+#include "ignite/odbc/query/query.h"
+#include "ignite/odbc/app/parameter_set.h"
+#include "ignite/odbc/cursor.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        /** Connection forward-declaration. */
+        class Connection;
+
+        namespace query
+        {
+            /**
+             * Query.
+             */
+            class BatchQuery : public Query
+            {
+            public:
+                /**
+                 * Constructor.
+                 *
+                 * @param diag Diagnostics collector.
+                 * @param connection Associated connection.
+                 * @param sql SQL query string.
+                 * @param params SQL params.
+                 */
+                BatchQuery(diagnostic::Diagnosable& diag, Connection& connection,
+                    const std::string& sql, const app::ParameterSet& params);
+
+                /**
+                 * Destructor.
+                 */
+                virtual ~BatchQuery();
+
+                /**
+                 * Execute query.
+                 *
+                 * @return True on success.
+                 */
+                virtual SqlResult Execute();
+
+                /**
+                 * Get column metadata.
+                 *
+                 * @return Column metadata.
+                 */
+                virtual const meta::ColumnMetaVector& GetMeta() const;
+
+                /**
+                 * Fetch next result row to application buffers.
+                 *
+                 * @param columnBindings Application buffers to put data to.
+                 * @return Operation result.
+                 */
+                virtual SqlResult FetchNextRow(app::ColumnBindingMap& columnBindings);
+                
+                /**
+                 * Get data of the specified column in the result set.
+                 *
+                 * @param columnIdx Column index.
+                 * @param buffer Buffer to put column data to.
+                 * @return Operation result.
+                 */
+                virtual SqlResult GetColumn(uint16_t columnIdx, app::ApplicationDataBuffer& buffer);
+
+                /**
+                 * Close query.
+                 *
+                 * @return Result.
+                 */
+                virtual SqlResult Close();
+
+                /**
+                 * Check if data is available.
+                 *
+                 * @return True if data is available.
+                 */
+                virtual bool DataAvailable() const;
+
+                /**
+                 * Get number of rows affected by the statement.
+                 *
+                 * @return Number of rows affected by the statement.
+                 */
+                virtual int64_t AffectedRows() const;
+
+                /**
+                 * Get SQL query string.
+                 *
+                 * @return SQL query string.
+                 */
+                const std::string& GetSql() const
+                {
+                    return sql;
+                }
+
+            private:
+                IGNITE_NO_COPY_ASSIGNMENT(BatchQuery);
+
+                /**
+                 * Make query execute request and use response to set internal
+                 * state.
+                 *
+                 * @param begin Paramset interval beginning.
+                 * @param end Paramset interval end.
+                 * @param last Last page flag.
+                 * @return Result.
+                 */
+                SqlResult MakeRequestExecuteBatch(SqlUlen begin, SqlUlen end, bool last);
+
+                /** Connection associated with the statement. */
+                Connection& connection;
+
+                /** SQL Query. */
+                std::string sql;
+
+                /** Parameter bindings. */
+                const app::ParameterSet& params;
+
+                /** Columns metadata. */
+                meta::ColumnMetaVector resultMeta;
+
+                /** Number of rows affected. */
+                int64_t rowsAffected;
+
+                /** Number of parameter sets successfully processed. */
+                int64_t setsProcessed;
+
+                /** Query executed. */
+                bool executed;
+
+                /** Data retrieved. */
+                bool dataRetrieved;
+            };
+        }
+    }
+}
+
+#endif //_IGNITE_ODBC_QUERY_BATCH_QUERY
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
index 68bb8776417c8..0424bf8364a6d 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
@@ -19,7 +19,7 @@
 #define _IGNITE_ODBC_QUERY_DATA_QUERY
 
 #include "ignite/odbc/query/query.h"
-#include "ignite/odbc/app/parameter.h"
+#include "ignite/odbc/app/parameter_set.h"
 #include "ignite/odbc/cursor.h"
 
 namespace ignite
@@ -46,7 +46,7 @@ namespace ignite
                  * @param params SQL params.
                  */
                 DataQuery(diagnostic::Diagnosable& diag, Connection& connection,
-                    const std::string& sql, const app::ParameterBindingMap& params);
+                    const std::string& sql, const app::ParameterSet& params);
 
                 /**
                  * Destructor.
@@ -122,24 +122,32 @@ namespace ignite
                  * Make query execute request and use response to set internal
                  * state.
                  *
-                 * @return True on success.
+                 * @return Result.
                  */
                 SqlResult MakeRequestExecute();
 
                 /**
                  * Make query close request.
                  *
-                 * @return True on success.
+                 * @return Result.
                  */
                 SqlResult MakeRequestClose();
 
                 /**
                  * Make data fetch request and use response to set internal state.
                  *
-                 * @return True on success.
+                 * @return Result.
                  */
                 SqlResult MakeRequestFetch();
 
+                /**
+                 * Close query.
+                 * Non-virtual implementation.
+                 *
+                 * @return True on success.
+                 */
+                SqlResult InternalClose();
+
                 /** Connection associated with the statement. */
                 Connection& connection;
 
@@ -147,7 +155,7 @@ namespace ignite
                 std::string sql;
 
                 /** Parameter bindings. */
-                const app::ParameterBindingMap& params;
+                const app::ParameterSet& params;
 
                 /** Columns metadata. */
                 meta::ColumnMetaVector resultMeta;
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
index 40be1ed4c8bd6..d8103af182a30 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
@@ -47,6 +47,9 @@ namespace ignite
                     /** Data query type. */
                     DATA,
 
+                    /** Batch query type. */
+                    BATCH,
+
                     /** Foreign keys query type. */
                     FOREIGN_KEYS,
 
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
index d337d0311b972..ee37bb012bb33 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
@@ -95,7 +95,7 @@ namespace ignite
                  * @return Number of rows affected by the statement.
                  */
                 virtual int64_t AffectedRows() const;
-                
+
             private:
                 IGNITE_NO_COPY_ASSIGNMENT(TypeInfoQuery);
 
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
index 596fc66e817a6..132cf78e45238 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
@@ -23,19 +23,12 @@
 #include <map>
 #include <memory>
 
-#include <ignite/impl/interop/interop_output_stream.h>
-#include <ignite/impl/interop/interop_input_stream.h>
-#include <ignite/impl/binary/binary_writer_impl.h>
-
 #include "ignite/odbc/meta/column_meta.h"
-#include "ignite/odbc/meta/table_meta.h"
 #include "ignite/odbc/query/query.h"
 #include "ignite/odbc/app/application_data_buffer.h"
-#include "ignite/odbc/app/parameter.h"
+#include "ignite/odbc/app/parameter_set.h"
 #include "ignite/odbc/diagnostic/diagnosable_adapter.h"
 #include "ignite/odbc/common_types.h"
-#include "ignite/odbc/cursor.h"
-#include "ignite/odbc/utility.h"
 
 namespace ignite
 {
@@ -102,7 +95,19 @@ namespace ignite
              * @param resLen A pointer to a buffer for the parameter's length.
              */
             void BindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType, int16_t paramSqlType,
-                               SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen);
+                SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen);
+
+            /**
+             * Unbind specified parameter.
+             *
+             * @param paramIdx Parameter index.
+             */
+            void UnbindParameter(uint16_t paramIdx);
+
+            /**
+             * Unbind all parameters.
+             */
+            void UnbindAllParameters();
 
             /**
              * Set statement attribute.
@@ -126,9 +131,9 @@ namespace ignite
             /**
              * Get number of binded parameters.
              *
-             * @return Number of binded parameters.
+             * @param paramNum Number of binded parameters.
              */
-            uint16_t GetParametersNumber();
+            void GetParametersNumber(uint16_t& paramNum);
 
             /**
              * Set parameter binding offset pointer.
@@ -137,13 +142,6 @@ namespace ignite
              */
             void SetParamBindOffsetPtr(int* ptr);
 
-            /**
-             * Get parameter binding offset pointer.
-             *
-             * @return Parameter binding offset pointer.
-             */
-            int* GetParamBindOffsetPtr();
-
             /**
              * Get value of the column in the result set.
              *
@@ -399,7 +397,14 @@ namespace ignite
              * Bind parameter.
              *
              * @param paramIdx Parameter index.
-             * @param param Parameter.
+             * @param ioType Type of the parameter (input/output).
+             * @param bufferType The data type of the parameter.
+             * @param paramSqlType The SQL data type of the parameter.
+             * @param columnSize  The size of the column or expression of the corresponding parameter marker.
+             * @param decDigits  The decimal digits of the column or expression of the corresponding parameter marker.
+             * @param buffer A pointer to a buffer for the parameter's data.
+             * @param bufferLen Length of the ParameterValuePtr buffer in bytes.
+             * @param resLen A pointer to a buffer for the parameter's length.
              */
             void SafeBindParameter(uint16_t paramIdx, const app::Parameter& param);
 
@@ -455,6 +460,15 @@ namespace ignite
              */
             SqlResult InternalGetAttribute(int attr, void* buf, SQLINTEGER bufLen, SQLINTEGER* valueLen);
 
+            /**
+             * Get number of binded parameters.
+             * Internal call.
+             *
+             * @param paramNum Number of binded parameters.
+             * @return Operation result.
+             */
+            SqlResult InternalGetParametersNumber(uint16_t& paramNum);
+
             /**
              * Get value of the column in the result set.
              *
@@ -681,12 +695,6 @@ namespace ignite
             /** Column bindings. */
             app::ColumnBindingMap columnBindings;
 
-            /** Parameter bindings. */
-            app::ParameterBindingMap paramBindings;
-
-            /** Parameter meta. */
-            std::vector<int8_t> paramTypes;
-
             /** Underlying query. */
             std::auto_ptr<query::Query> currentQuery;
 
@@ -696,14 +704,10 @@ namespace ignite
             /** Array to store statuses of rows fetched by the last fetch. */
             uint16_t* rowStatuses;
 
-            /** Offset added to pointers to change binding of parameters. */
-            int* paramBindOffset;
-
             /** Offset added to pointers to change binding of column data. */
             int* columnBindOffset;
 
-            /** Index of the parameter, which is currently being set. */
-            uint16_t currentParamIdx;
+            app::ParameterSet parameters;
         };
     }
 }
diff --git a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
index b00e4326e8671..e47f618aa83ad 100644
--- a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
+++ b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
@@ -160,6 +160,7 @@
     <ClCompile Include="..\..\os\win\src\system\ui\window.cpp" />
     <ClCompile Include="..\..\src\app\application_data_buffer.cpp" />
     <ClCompile Include="..\..\src\app\parameter.cpp" />
+    <ClCompile Include="..\..\src\app\parameter_set.cpp" />
     <ClCompile Include="..\..\src\column.cpp" />
     <ClCompile Include="..\..\src\common_types.cpp" />
     <ClCompile Include="..\..\src\config\configuration.cpp" />
@@ -172,10 +173,12 @@
     <ClCompile Include="..\..\src\dsn_config.cpp" />
     <ClCompile Include="..\..\src\entry_points.cpp" />
     <ClCompile Include="..\..\src\environment.cpp" />
+    <ClCompile Include="..\..\src\message.cpp" />
     <ClCompile Include="..\..\src\meta\column_meta.cpp" />
     <ClCompile Include="..\..\src\meta\table_meta.cpp" />
     <ClCompile Include="..\..\src\odbc.cpp" />
     <ClCompile Include="..\..\src\protocol_version.cpp" />
+    <ClCompile Include="..\..\src\query\batch_query.cpp" />
     <ClCompile Include="..\..\src\query\data_query.cpp" />
     <ClCompile Include="..\..\src\query\column_metadata_query.cpp" />
     <ClCompile Include="..\..\src\query\foreign_keys_query.cpp" />
@@ -197,6 +200,7 @@
     <ClInclude Include="..\..\include\ignite\odbc.h" />
     <ClInclude Include="..\..\include\ignite\odbc\app\application_data_buffer.h" />
     <ClInclude Include="..\..\include\ignite\odbc\app\parameter.h" />
+    <ClInclude Include="..\..\include\ignite\odbc\app\parameter_set.h" />
     <ClInclude Include="..\..\include\ignite\odbc\column.h" />
     <ClInclude Include="..\..\include\ignite\odbc\common_types.h" />
     <ClInclude Include="..\..\include\ignite\odbc\config\configuration.h" />
@@ -215,6 +219,7 @@
     <ClInclude Include="..\..\include\ignite\odbc\meta\table_meta.h" />
     <ClInclude Include="..\..\include\ignite\odbc\parser.h" />
     <ClInclude Include="..\..\include\ignite\odbc\protocol_version.h" />
+    <ClInclude Include="..\..\include\ignite\odbc\query\batch_query.h" />
     <ClInclude Include="..\..\include\ignite\odbc\query\data_query.h" />
     <ClInclude Include="..\..\include\ignite\odbc\query\column_metadata_query.h" />
     <ClInclude Include="..\..\include\ignite\odbc\query\foreign_keys_query.h" />
diff --git a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
index 1828be1028382..e252d5d6c14dd 100644
--- a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
+++ b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
@@ -139,6 +139,15 @@
     <ClCompile Include="..\..\src\protocol_version.cpp">
       <Filter>Code</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\app\parameter_set.cpp">
+      <Filter>Code\app</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\query\batch_query.cpp">
+      <Filter>Code\query</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\message.cpp">
+      <Filter>Code</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="module.def">
@@ -266,5 +275,11 @@
     <ClInclude Include="..\..\include\ignite\odbc\protocol_version.h">
       <Filter>Code</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\ignite\odbc\app\parameter_set.h">
+      <Filter>Code\app</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\include\ignite\odbc\query\batch_query.h">
+      <Filter>Code\query</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp b/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp
index 026cd60f79206..bb9e58498f763 100644
--- a/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp
+++ b/modules/platforms/cpp/odbc/src/app/application_data_buffer.cpp
@@ -33,35 +33,38 @@ namespace ignite
     {
         namespace app
         {
-            using ignite::impl::binary::BinaryUtils;
+            using impl::binary::BinaryUtils;
 
             ApplicationDataBuffer::ApplicationDataBuffer() :
                 type(type_traits::IGNITE_ODBC_C_TYPE_UNSUPPORTED),
                 buffer(0),
                 buflen(0),
                 reslen(0),
-                offset(0)
+                byteOffset(0),
+                elementOffset(0)
             {
                 // No-op.
             }
 
             ApplicationDataBuffer::ApplicationDataBuffer(type_traits::IgniteSqlType type,
-                void* buffer, SqlLen buflen, SqlLen* reslen, int** offset) :
+                void* buffer, SqlLen buflen, SqlLen* reslen) :
                 type(type),
                 buffer(buffer),
                 buflen(buflen),
                 reslen(reslen),
-                offset(offset)
+                byteOffset(0),
+                elementOffset(0)
             {
                 // No-op.
             }
 
-            ApplicationDataBuffer::ApplicationDataBuffer(const ApplicationDataBuffer & other) :
+            ApplicationDataBuffer::ApplicationDataBuffer(const ApplicationDataBuffer& other) :
                 type(other.type),
                 buffer(other.buffer),
                 buflen(other.buflen),
                 reslen(other.reslen),
-                offset(other.offset)
+                byteOffset(other.byteOffset),
+                elementOffset(other.elementOffset)
             {
                 // No-op.
             }
@@ -77,7 +80,8 @@ namespace ignite
                 buffer = other.buffer;
                 buflen = other.buflen;
                 reslen = other.reslen;
-                offset = other.offset;
+                byteOffset = other.byteOffset;
+                elementOffset = other.elementOffset;
 
                 return *this;
             }
@@ -1100,22 +1104,22 @@ namespace ignite
 
             const void* ApplicationDataBuffer::GetData() const
             {
-                return ApplyOffset(buffer);
+                return ApplyOffset(buffer, GetElementSize());
             }
 
             const SqlLen* ApplicationDataBuffer::GetResLen() const
             {
-                return ApplyOffset(reslen);
+                return ApplyOffset(reslen, sizeof(*reslen));
             }
 
             void* ApplicationDataBuffer::GetData()
             {
-                return ApplyOffset(buffer);
+                return ApplyOffset(buffer, GetElementSize());
             }
 
             SqlLen* ApplicationDataBuffer::GetResLen()
             {
-                return ApplyOffset(reslen);
+                return ApplyOffset(reslen, sizeof(*reslen));
             }
 
             template<typename T>
@@ -1461,12 +1465,12 @@ namespace ignite
             }
 
             template<typename T>
-            T* ApplicationDataBuffer::ApplyOffset(T* ptr) const
+            T* ApplicationDataBuffer::ApplyOffset(T* ptr, size_t elemSize) const
             {
-                if (!ptr || !offset || !*offset)
+                if (!ptr)
                     return ptr;
 
-                return utility::GetPointerWithOffset(ptr, **offset);
+                return utility::GetPointerWithOffset(ptr, byteOffset + elemSize * elementOffset);
             }
 
             bool ApplicationDataBuffer::IsDataAtExec() const
@@ -1556,6 +1560,64 @@ namespace ignite
                 return 0;
             }
 
+            SqlLen ApplicationDataBuffer::GetElementSize() const
+            {
+                using namespace type_traits;
+
+                switch (type)
+                {
+                    case IGNITE_ODBC_C_TYPE_WCHAR:
+                    case IGNITE_ODBC_C_TYPE_CHAR:
+                    case IGNITE_ODBC_C_TYPE_BINARY:
+                        return buflen;
+
+                    case IGNITE_ODBC_C_TYPE_SIGNED_SHORT:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT:
+                        return static_cast<SqlLen>(sizeof(short));
+
+                    case IGNITE_ODBC_C_TYPE_SIGNED_LONG:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_LONG:
+                        return static_cast<SqlLen>(sizeof(long));
+
+                    case IGNITE_ODBC_C_TYPE_FLOAT:
+                        return static_cast<SqlLen>(sizeof(float));
+
+                    case IGNITE_ODBC_C_TYPE_DOUBLE:
+                        return static_cast<SqlLen>(sizeof(double));
+
+                    case IGNITE_ODBC_C_TYPE_BIT:
+                    case IGNITE_ODBC_C_TYPE_SIGNED_TINYINT:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_TINYINT:
+                        return static_cast<SqlLen>(sizeof(char));
+
+                    case IGNITE_ODBC_C_TYPE_SIGNED_BIGINT:
+                    case IGNITE_ODBC_C_TYPE_UNSIGNED_BIGINT:
+                        return static_cast<SqlLen>(sizeof(SQLBIGINT));
+
+                    case IGNITE_ODBC_C_TYPE_TDATE:
+                        return static_cast<SqlLen>(sizeof(SQL_DATE_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_TTIME:
+                        return static_cast<SqlLen>(sizeof(SQL_TIME_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_TTIMESTAMP:
+                        return static_cast<SqlLen>(sizeof(SQL_TIMESTAMP_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_NUMERIC:
+                        return static_cast<SqlLen>(sizeof(SQL_NUMERIC_STRUCT));
+
+                    case IGNITE_ODBC_C_TYPE_GUID:
+                        return static_cast<SqlLen>(sizeof(SQLGUID));
+
+                    case IGNITE_ODBC_C_TYPE_DEFAULT:
+                    case IGNITE_ODBC_C_TYPE_UNSUPPORTED:
+                    default:
+                        break;
+                }
+
+                return 0;
+            }
+
             SqlLen ApplicationDataBuffer::GetInputSize() const
             {
                 if (!IsDataAtExec())
diff --git a/modules/platforms/cpp/odbc/src/app/parameter.cpp b/modules/platforms/cpp/odbc/src/app/parameter.cpp
index ded2e4bd29072..083b744c04686 100644
--- a/modules/platforms/cpp/odbc/src/app/parameter.cpp
+++ b/modules/platforms/cpp/odbc/src/app/parameter.cpp
@@ -16,8 +16,6 @@
  */
 
 #include <algorithm>
-#include <string>
-#include <sstream>
 
 #include "ignite/odbc/system/odbc_constants.h"
 #include "ignite/odbc/app/parameter.h"
@@ -78,7 +76,7 @@ namespace ignite
                 return *this;
             }
 
-            void Parameter::Write(ignite::impl::binary::BinaryWriterImpl& writer) const
+            void Parameter::Write(impl::binary::BinaryWriterImpl& writer, int offset, SqlUlen idx) const
             {
                 if (buffer.GetInputSize() == SQL_NULL_DATA)
                 {
@@ -89,6 +87,8 @@ namespace ignite
 
                 // Buffer to use to get data.
                 ApplicationDataBuffer buf(buffer);
+                buf.SetByteOffset(offset);
+                buf.SetElementOffset(idx);
 
                 SqlLen storedDataLen = static_cast<SqlLen>(storedData.size());
 
@@ -209,6 +209,11 @@ namespace ignite
                 return buffer;
             }
 
+            const ApplicationDataBuffer& Parameter::GetBuffer() const
+            {
+                return buffer;
+            }
+
             void Parameter::ResetStoredData()
             {
                 storedData.clear();
diff --git a/modules/platforms/cpp/odbc/src/app/parameter_set.cpp b/modules/platforms/cpp/odbc/src/app/parameter_set.cpp
new file mode 100644
index 0000000000000..c110d05dc5b38
--- /dev/null
+++ b/modules/platforms/cpp/odbc/src/app/parameter_set.cpp
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ignite/odbc/app/parameter_set.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        namespace app
+        {
+            ParameterSet::ParameterSet():
+                parameters(),
+                paramTypes(),
+                paramBindOffset(0),
+                processedParamRows(0),
+                paramSetSize(1),
+                paramSetPos(0),
+                currentParamIdx(0),
+                typesSet(false)
+            {
+                // No-op.
+            }
+
+            void ParameterSet::SetParamSetSize(SqlUlen size)
+            {
+                paramSetSize = size;
+            }
+
+            void ParameterSet::BindParameter(uint16_t paramIdx, const Parameter& param)
+            {
+                parameters[paramIdx] = param;
+            }
+
+            void ParameterSet::UnbindParameter(uint16_t paramIdx)
+            {
+                parameters.erase(paramIdx);
+            }
+
+            void ParameterSet::UnbindAll()
+            {
+                parameters.clear();
+            }
+
+            uint16_t ParameterSet::GetParametersNumber() const
+            {
+                return static_cast<uint16_t>(parameters.size());
+            }
+
+            void ParameterSet::SetParamBindOffsetPtr(int* ptr)
+            {
+                paramBindOffset = ptr;
+            }
+
+            int* ParameterSet::GetParamBindOffsetPtr()
+            {
+                return paramBindOffset;
+            }
+
+            void ParameterSet::Prepare()
+            {
+                paramTypes.clear();
+
+                typesSet = false;
+
+                paramSetPos = 0;
+
+                for (ParameterBindingMap::iterator it = parameters.begin(); it != parameters.end(); ++it)
+                    it->second.ResetStoredData();
+            }
+
+            bool ParameterSet::IsDataAtExecNeeded() const
+            {
+                for (ParameterBindingMap::const_iterator it = parameters.begin(); it != parameters.end(); ++it)
+                {
+                    if (!it->second.IsDataReady())
+                        return true;
+                }
+
+                return false;
+            }
+
+            void ParameterSet::SetParamsProcessedPtr(SqlUlen* ptr)
+            {
+                processedParamRows = ptr;
+            }
+
+            SqlUlen* ParameterSet::GetParamsProcessedPtr()
+            {
+                return processedParamRows;
+            }
+
+            void ParameterSet::SetParamsProcessed(SqlUlen processed) const
+            {
+                if (processedParamRows)
+                    *processedParamRows = processed;
+            }
+
+            void ParameterSet::UpdateParamsTypes(const ParameterTypeVector& meta)
+            {
+                paramTypes = meta;
+
+                typesSet = true;
+            }
+
+            int8_t ParameterSet::GetParamType(int16_t idx, int8_t dflt)
+            {
+                if (idx > 0 && static_cast<size_t>(idx) <= paramTypes.size())
+                    return paramTypes[idx - 1];
+
+                return dflt;
+            }
+
+            uint16_t ParameterSet::GetExpectedParamNum()
+            {
+                return static_cast<uint16_t>(paramTypes.size());
+            }
+
+            bool ParameterSet::IsMetadataSet() const
+            {
+                return typesSet;
+            }
+
+            bool ParameterSet::IsParameterSelected() const
+            {
+                return currentParamIdx != 0;
+            }
+
+            Parameter* ParameterSet::GetParameter(uint16_t idx)
+            {
+                ParameterBindingMap::iterator it = parameters.find(currentParamIdx);
+
+                if (it != parameters.end())
+                    return &it->second;
+
+                return 0;
+            }
+
+            Parameter* ParameterSet::GetSelectedParameter()
+            {
+                return GetParameter(currentParamIdx);
+            }
+
+            Parameter* ParameterSet::SelectNextParameter()
+            {
+                for (ParameterBindingMap::iterator it = parameters.begin(); it != parameters.end(); ++it)
+                {
+                    uint16_t paramIdx = it->first;
+                    Parameter& param = it->second;
+
+                    if (!param.IsDataReady())
+                    {
+                        currentParamIdx = paramIdx;
+
+                        return &param;
+                    }
+                }
+
+                return 0;
+            }
+
+            void ParameterSet::Write(impl::binary::BinaryWriterImpl& writer) const
+            {
+                writer.WriteInt32(CalculateRowLen());
+
+                WriteRow(writer, 0);
+            }
+
+            void ParameterSet::Write(impl::binary::BinaryWriterImpl& writer, SqlUlen begin, SqlUlen end, bool last) const
+            {
+                int32_t rowLen = CalculateRowLen();
+
+                writer.WriteInt32(rowLen);
+
+                SqlUlen intervalEnd = std::min(paramSetSize, end);
+
+                assert(begin < intervalEnd);
+
+                int32_t intervalLen = static_cast<int32_t>(intervalEnd - begin);
+
+                writer.WriteInt32(intervalLen);
+                writer.WriteBool(last);
+
+                if (rowLen)
+                {
+                    for (SqlUlen i = begin; i < intervalEnd; ++i)
+                        WriteRow(writer, i);
+                }
+            }
+
+            void ParameterSet::WriteRow(impl::binary::BinaryWriterImpl& writer, SqlUlen idx) const
+            {
+                uint16_t prev = 0;
+
+                int appOffset = paramBindOffset ? *paramBindOffset : 0;
+
+                for (ParameterBindingMap::const_iterator it = parameters.begin(); it != parameters.end(); ++it)
+                {
+                    uint16_t paramIdx = it->first;
+                    const Parameter& param = it->second;
+
+                    while ((paramIdx - prev) > 1)
+                    {
+                        writer.WriteNull();
+                        ++prev;
+                    }
+
+                    param.Write(writer, appOffset, idx);
+
+                    prev = paramIdx;
+                }
+            }
+
+            int32_t ParameterSet::CalculateRowLen() const
+            {
+                if (!parameters.empty())
+                    return static_cast<int32_t>(parameters.rbegin()->first);
+
+                return  0;
+            }
+
+            int32_t ParameterSet::GetParamSetSize() const
+            {
+                return static_cast<int32_t>(paramSetSize);
+            }
+        }
+    }
+}
diff --git a/modules/platforms/cpp/odbc/src/config/connection_info.cpp b/modules/platforms/cpp/odbc/src/config/connection_info.cpp
index a34d43447e4f2..a74f23cb65e7d 100644
--- a/modules/platforms/cpp/odbc/src/config/connection_info.cpp
+++ b/modules/platforms/cpp/odbc/src/config/connection_info.cpp
@@ -118,6 +118,8 @@ namespace ignite
                     DBG_STR_CASE(SQL_CONVERT_WLONGVARCHAR);
                     DBG_STR_CASE(SQL_CONVERT_WVARCHAR);
                     DBG_STR_CASE(SQL_CONVERT_GUID);
+                    DBG_STR_CASE(SQL_PARAM_ARRAY_ROW_COUNTS);
+                    DBG_STR_CASE(SQL_PARAM_ARRAY_SELECTS);
                 default:
                     break;
                 }
@@ -592,6 +594,40 @@ namespace ignite
                     SQL_CVT_BINARY | SQL_CVT_VARBINARY | SQL_CVT_LONGVARBINARY | SQL_CVT_GUID;
 #endif //SQL_CONVERT_GUID
 
+#ifdef SQL_PARAM_ARRAY_ROW_COUNTS
+                // Enumerating the driver's properties regarding the availability of row counts in a parameterized
+                // execution. Has the following values:
+                //
+                // SQL_PARC_BATCH = Individual row counts are available for each set of parameters. This is conceptually
+                //     equivalent to the driver generating a batch of SQL statements, one for each parameter set in the
+                //     array. Extended error information can be retrieved by using the SQL_PARAM_STATUS_PTR descriptor
+                //     field.
+                //
+                // SQL_PARC_NO_BATCH = There is only one row count available, which is the cumulative row count
+                //     resulting from the execution of the statement for the entire array of parameters. This is
+                //     conceptually equivalent to treating the statement together with the complete parameter array as
+                //     one atomic unit. Errors are handled the same as if one statement were executed.
+                intParams[SQL_PARAM_ARRAY_ROW_COUNTS] = SQL_PARC_NO_BATCH;
+#endif //SQL_PARAM_ARRAY_ROW_COUNTS
+
+#ifdef SQL_PARAM_ARRAY_SELECTS
+                // Enumerating the driver's properties regarding the availability of result sets in a parameterized
+                // execution. Has the following values:
+                //
+                // SQL_PAS_BATCH = There is one result set available per set of parameters. This is conceptually
+                //     equivalent to the driver generating a batch of SQL statements, one for each parameter set in
+                //     the array.
+                //
+                // SQL_PAS_NO_BATCH = There is only one result set available, which represents the cumulative result set
+                //     resulting from the execution of the statement for the complete array of parameters. This is
+                //     conceptually equivalent to treating the statement together with the complete parameter array as
+                //     one atomic unit.
+                //
+                // SQL_PAS_NO_SELECT = A driver does not allow a result - set generating statement to be executed with
+                //     an array of parameters.
+                intParams[SQL_PARAM_ARRAY_SELECTS] = SQL_PAS_NO_SELECT;
+#endif //SQL_PARAM_ARRAY_SELECTS
+
                 //======================= Short Params ========================
 #ifdef SQL_MAX_CONCURRENT_ACTIVITIES
                 // The maximum number of active statements that the driver can
diff --git a/modules/platforms/cpp/odbc/src/message.cpp b/modules/platforms/cpp/odbc/src/message.cpp
new file mode 100644
index 0000000000000..741540526fb19
--- /dev/null
+++ b/modules/platforms/cpp/odbc/src/message.cpp
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ignite/odbc/message.h"
+#include "ignite/odbc/utility.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        HandshakeRequest::HandshakeRequest(int64_t version, bool distributedJoins, bool enforceJoinOrder):
+            version(version),
+            distributedJoins(distributedJoins),
+            enforceJoinOrder(enforceJoinOrder)
+        {
+            // No-op.
+        }
+
+        HandshakeRequest::~HandshakeRequest()
+        {
+            // No-op.
+        }
+
+        void HandshakeRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_HANDSHAKE);
+
+            writer.WriteInt64(version);
+
+            writer.WriteBool(distributedJoins);
+            writer.WriteBool(enforceJoinOrder);
+        }
+
+        QueryExecuteRequest::QueryExecuteRequest(const std::string& schema, const std::string& sql, const app::ParameterSet& params):
+            cache(schema),
+            sql(sql),
+            params(params)
+        {
+            // No-op.
+        }
+
+        QueryExecuteRequest::~QueryExecuteRequest()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_EXECUTE_SQL_QUERY);
+
+            if (cache.empty())
+                writer.WriteNull();
+            else
+                writer.WriteObject<std::string>(cache);
+
+            writer.WriteObject<std::string>(sql);
+
+            params.Write(writer);
+        }
+
+        QueryExecuteBatchtRequest::QueryExecuteBatchtRequest(const std::string& schema, const std::string& sql,
+            const app::ParameterSet& params, SqlUlen begin, SqlUlen end, bool last):
+            schema(schema),
+            sql(sql),
+            params(params),
+            begin(begin),
+            end(end),
+            last(last)
+        {
+            // No-op.
+        }
+
+        QueryExecuteBatchtRequest::~QueryExecuteBatchtRequest()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteBatchtRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_EXECUTE_SQL_QUERY_BATCH);
+
+            if (schema.empty())
+                writer.WriteNull();
+            else
+                writer.WriteObject<std::string>(schema);
+
+            writer.WriteObject<std::string>(sql);
+
+            params.Write(writer, begin, end, last);
+        }
+
+        QueryCloseRequest::QueryCloseRequest(int64_t queryId): queryId(queryId)
+        {
+            // No-op.
+        }
+
+        QueryCloseRequest::~QueryCloseRequest()
+        {
+            // No-op.
+        }
+
+        void QueryCloseRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_CLOSE_SQL_QUERY);
+            writer.WriteInt64(queryId);
+        }
+
+        QueryFetchRequest::QueryFetchRequest(int64_t queryId, int32_t pageSize):
+            queryId(queryId),
+            pageSize(pageSize)
+        {
+            // No-op.
+        }
+
+        QueryFetchRequest::~QueryFetchRequest()
+        {
+            // No-op.
+        }
+
+        void QueryFetchRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_FETCH_SQL_QUERY);
+            writer.WriteInt64(queryId);
+            writer.WriteInt32(pageSize);
+        }
+
+        QueryGetColumnsMetaRequest::QueryGetColumnsMetaRequest(const std::string& schema, const std::string& table, const std::string& column):
+            schema(schema),
+            table(table),
+            column(column)
+        {
+            // No-op.
+        }
+
+        QueryGetColumnsMetaRequest::~QueryGetColumnsMetaRequest()
+        {
+            // No-op.
+        }
+
+        void QueryGetColumnsMetaRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_GET_COLUMNS_METADATA);
+
+            utility::WriteString(writer, schema);
+            utility::WriteString(writer, table);
+            utility::WriteString(writer, column);
+        }
+
+        QueryGetTablesMetaRequest::QueryGetTablesMetaRequest(const std::string& catalog, const std::string& schema, const std::string& table, const std::string& tableTypes):
+            catalog(catalog),
+            schema(schema),
+            table(table),
+            tableTypes(tableTypes)
+        {
+            // No-op.
+        }
+
+        QueryGetTablesMetaRequest::~QueryGetTablesMetaRequest()
+        {
+            // No-op.
+        }
+
+        void QueryGetTablesMetaRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_GET_TABLES_METADATA);
+
+            utility::WriteString(writer, catalog);
+            utility::WriteString(writer, schema);
+            utility::WriteString(writer, table);
+            utility::WriteString(writer, tableTypes);
+        }
+
+        void QueryGetParamsMetaRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+        {
+            writer.WriteInt8(REQUEST_TYPE_GET_PARAMS_METADATA);
+
+            utility::WriteString(writer, cacheName);
+            utility::WriteString(writer, sqlQuery);
+        }
+
+        Response::Response():
+            status(RESPONSE_STATUS_FAILED),
+            error()
+        {
+            // No-op.
+        }
+
+        Response::~Response()
+        {
+            // No-op.
+        }
+
+        void Response::Read(impl::binary::BinaryReaderImpl& reader)
+        {
+            status = reader.ReadInt8();
+
+            if (status == RESPONSE_STATUS_SUCCESS)
+                ReadOnSuccess(reader);
+            else
+                utility::ReadString(reader, error);
+        }
+
+        void Response::ReadOnSuccess(impl::binary::BinaryReaderImpl&)
+        {
+            // No-op.
+        }
+
+        HandshakeResponse::HandshakeResponse():
+            accepted(false),
+            protoVerSince(),
+            currentVer()
+        {
+            // No-op.
+        }
+
+        HandshakeResponse::~HandshakeResponse()
+        {
+            // No-op.
+        }
+
+        void HandshakeResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            accepted = reader.ReadBool();
+
+            if (!accepted)
+            {
+                utility::ReadString(reader, protoVerSince);
+                utility::ReadString(reader, currentVer);
+            }
+        }
+
+        QueryCloseResponse::QueryCloseResponse(): queryId(0)
+        {
+            // No-op.
+        }
+
+        QueryCloseResponse::~QueryCloseResponse()
+        {
+            // No-op.
+        }
+
+        void QueryCloseResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            queryId = reader.ReadInt64();
+        }
+
+        QueryExecuteResponse::QueryExecuteResponse(): queryId(0), meta()
+        {
+            // No-op.
+        }
+
+        QueryExecuteResponse::~QueryExecuteResponse()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            queryId = reader.ReadInt64();
+
+            meta::ReadColumnMetaVector(reader, meta);
+        }
+
+        QueryExecuteBatchResponse::QueryExecuteBatchResponse():
+            affectedRows(0),
+            errorSetIdx(-1),
+            errorMessage()
+        {
+            // No-op.
+        }
+
+        QueryExecuteBatchResponse::~QueryExecuteBatchResponse()
+        {
+            // No-op.
+        }
+
+        void QueryExecuteBatchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            bool success = reader.ReadBool();
+            affectedRows = reader.ReadInt64();
+
+            if (!success)
+            {
+                errorSetIdx = reader.ReadInt64();
+                errorMessage = reader.ReadObject<std::string>();
+            }
+        }
+
+        QueryFetchResponse::QueryFetchResponse(ResultPage& resultPage): queryId(0), resultPage(resultPage)
+        {
+            // No-op.
+        }
+
+        QueryFetchResponse::~QueryFetchResponse()
+        {
+            // No-op.
+        }
+
+        void QueryFetchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            queryId = reader.ReadInt64();
+
+            resultPage.Read(reader);
+        }
+
+        QueryGetColumnsMetaResponse::QueryGetColumnsMetaResponse()
+        {
+            // No-op.
+        }
+
+        QueryGetColumnsMetaResponse::~QueryGetColumnsMetaResponse()
+        {
+            // No-op.
+        }
+
+        void QueryGetColumnsMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            meta::ReadColumnMetaVector(reader, meta);
+        }
+
+        QueryGetTablesMetaResponse::QueryGetTablesMetaResponse()
+        {
+            // No-op.
+        }
+
+        QueryGetTablesMetaResponse::~QueryGetTablesMetaResponse()
+        {
+            // No-op.
+        }
+
+        void QueryGetTablesMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            meta::ReadTableMetaVector(reader, meta);
+        }
+
+        QueryGetParamsMetaResponse::QueryGetParamsMetaResponse()
+        {
+            // No-op.
+        }
+
+        QueryGetParamsMetaResponse::~QueryGetParamsMetaResponse()
+        {
+            // No-op.
+        }
+
+        void QueryGetParamsMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+        {
+            utility::ReadByteArray(reader, typeIds);
+        }
+    }
+}
+
diff --git a/modules/platforms/cpp/odbc/src/odbc.cpp b/modules/platforms/cpp/odbc/src/odbc.cpp
index 542e64cca668f..af8c73489902c 100644
--- a/modules/platforms/cpp/odbc/src/odbc.cpp
+++ b/modules/platforms/cpp/odbc/src/odbc.cpp
@@ -417,9 +417,9 @@ namespace ignite
         using odbc::Statement;
         using odbc::app::ApplicationDataBuffer;
 
-        LOG_MSG("SQLBindCol called: index=" << colNum << ", type=" << targetType << 
-                ", targetValue=" << reinterpret_cast<size_t>(targetValue) << 
-                ", bufferLength=" << bufferLength << 
+        LOG_MSG("SQLBindCol called: index=" << colNum << ", type=" << targetType <<
+                ", targetValue=" << reinterpret_cast<size_t>(targetValue) <<
+                ", bufferLength=" << bufferLength <<
                 ", lengthInd=" << reinterpret_cast<size_t>(strLengthOrIndicator));
 
         Statement *statement = reinterpret_cast<Statement*>(stmt);
@@ -618,7 +618,7 @@ namespace ignite
         if (!statement)
             return SQL_INVALID_HANDLE;
 
-        statement->BindParameter(paramIdx, ioType, bufferType, paramSqlType, columnSize, decDigits, buffer, bufferLen, resLen);
+        statement->BindParameter(paramIdx,ioType , bufferType, paramSqlType, columnSize, decDigits, buffer, bufferLen, resLen);
 
         return statement->GetDiagnosticRecords().GetReturnCode();
     }
@@ -897,7 +897,13 @@ namespace ignite
             return SQL_INVALID_HANDLE;
 
         if (paramCnt)
-            *paramCnt = static_cast<SQLSMALLINT>(statement->GetParametersNumber());
+        {
+            uint16_t num = 0;
+
+            statement->GetParametersNumber(num);
+
+            *paramCnt = static_cast<SQLSMALLINT>(num);
+        }
 
         return statement->GetDiagnosticRecords().GetReturnCode();
     }
diff --git a/modules/platforms/cpp/odbc/src/query/batch_query.cpp b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
new file mode 100644
index 0000000000000..dd929026750b1
--- /dev/null
+++ b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ignite/odbc/connection.h"
+#include "ignite/odbc/message.h"
+#include "ignite/odbc/query/batch_query.h"
+#include "ignite/odbc/log.h"
+
+namespace ignite
+{
+    namespace odbc
+    {
+        namespace query
+        {
+            BatchQuery::BatchQuery(diagnostic::Diagnosable& diag, Connection& connection,
+                const std::string& sql, const app::ParameterSet& params) :
+                Query(diag, BATCH),
+                connection(connection),
+                sql(sql),
+                params(params),
+                resultMeta(),
+                rowsAffected(0),
+                setsProcessed(0),
+                executed(false),
+                dataRetrieved(false)
+            {
+                // No-op.
+            }
+
+            BatchQuery::~BatchQuery()
+            {
+                // No-op.
+            }
+
+            SqlResult BatchQuery::Execute()
+            {
+                if (executed)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query cursor is in open state already.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                int32_t maxPageSize = connection.GetConfiguration().GetPageSize();
+                int32_t rowNum = params.GetParamSetSize();
+                SqlResult res;
+
+                int32_t processed = 0;
+
+                do {
+                    int32_t currentPageSize = std::min(maxPageSize, rowNum - processed);
+                    bool lastPage = currentPageSize == rowNum - processed;
+
+                    res = MakeRequestExecuteBatch(processed, processed + currentPageSize, lastPage);
+
+                    processed += currentPageSize;
+                } while (res == SQL_RESULT_SUCCESS && processed < rowNum);
+
+                params.SetParamsProcessed(static_cast<SqlUlen>(setsProcessed));
+
+                return res;
+            }
+
+            const meta::ColumnMetaVector& BatchQuery::GetMeta() const
+            {
+                return resultMeta;
+            }
+
+            SqlResult BatchQuery::FetchNextRow(app::ColumnBindingMap& columnBindings)
+            {
+                if (!executed)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query was not executed.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                if (dataRetrieved)
+                    return SQL_RESULT_NO_DATA;
+
+                app::ColumnBindingMap::iterator it = columnBindings.find(1);
+
+                if (it != columnBindings.end())
+                    it->second.PutInt64(rowsAffected);
+
+                dataRetrieved = true;
+
+                return SQL_RESULT_SUCCESS;
+            }
+
+            SqlResult BatchQuery::GetColumn(uint16_t columnIdx, app::ApplicationDataBuffer& buffer)
+            {
+                if (!executed)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query was not executed.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                if (dataRetrieved)
+                    return SQL_RESULT_NO_DATA;
+
+                if (columnIdx != 1)
+                {
+                    std::stringstream builder;
+                    builder << "Column with id " << columnIdx << " is not available in result set.";
+
+                    diag.AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR, builder.str());
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                buffer.PutInt64(rowsAffected);
+
+                return SQL_RESULT_SUCCESS;
+            }
+
+            SqlResult BatchQuery::Close()
+            {
+                return SQL_RESULT_SUCCESS;
+            }
+
+            bool BatchQuery::DataAvailable() const
+            {
+                return false;
+            }
+
+            int64_t BatchQuery::AffectedRows() const
+            {
+                return rowsAffected;
+            }
+
+            SqlResult BatchQuery::MakeRequestExecuteBatch(SqlUlen begin, SqlUlen end, bool last)
+            {
+                const std::string& schema = connection.GetCache();
+
+                QueryExecuteBatchtRequest req(schema, sql, params, begin, end, last);
+                QueryExecuteBatchResponse rsp;
+
+                try
+                {
+                    connection.SyncMessage(req, rsp);
+                }
+                catch (const IgniteError& err)
+                {
+                    diag.AddStatusRecord(SQL_STATE_HYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                if (rsp.GetStatus() != RESPONSE_STATUS_SUCCESS)
+                {
+                    LOG_MSG("Error: " << rsp.GetError());
+
+                    diag.AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR, rsp.GetError());
+
+                    return SQL_RESULT_ERROR;
+                }
+
+                rowsAffected += rsp.GetAffectedRows();
+                LOG_MSG("rowsAffected: " << rowsAffected);
+
+                if (!rsp.GetErrorMessage().empty())
+                {
+                    LOG_MSG("Error: " << rsp.GetErrorMessage());
+
+                    setsProcessed += rsp.GetErrorSetIdx();
+                    LOG_MSG("setsProcessed: " << setsProcessed);
+
+                    diag.AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR, rsp.GetErrorMessage(),
+                        static_cast<int32_t>(setsProcessed), 0);
+
+                    return SQL_RESULT_SUCCESS_WITH_INFO;
+                }
+
+                setsProcessed += end - begin;
+                LOG_MSG("setsProcessed: " << setsProcessed);
+
+                return SQL_RESULT_SUCCESS;
+            }
+        }
+    }
+}
+
diff --git a/modules/platforms/cpp/odbc/src/query/data_query.cpp b/modules/platforms/cpp/odbc/src/query/data_query.cpp
index c9762ad2aaa14..f2ee7ec4758ad 100644
--- a/modules/platforms/cpp/odbc/src/query/data_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/data_query.cpp
@@ -19,6 +19,7 @@
 #include "ignite/odbc/message.h"
 #include "ignite/odbc/log.h"
 #include "ignite/odbc/query/data_query.h"
+#include "ignite/odbc/query/batch_query.h"
 
 namespace ignite
 {
@@ -26,9 +27,8 @@ namespace ignite
     {
         namespace query
         {
-            DataQuery::DataQuery(diagnostic::Diagnosable& diag,
-                Connection& connection, const std::string& sql,
-                const app::ParameterBindingMap& params) :
+            DataQuery::DataQuery(diagnostic::Diagnosable& diag, Connection& connection,
+                const std::string& sql, const app::ParameterSet& params) :
                 Query(diag, DATA),
                 connection(connection),
                 sql(sql),
@@ -39,7 +39,7 @@ namespace ignite
 
             DataQuery::~DataQuery()
             {
-                Close();
+                InternalClose();
             }
 
             SqlResult DataQuery::Execute()
@@ -140,6 +140,11 @@ namespace ignite
             }
 
             SqlResult DataQuery::Close()
+            {
+                return InternalClose();
+            }
+
+            SqlResult DataQuery::InternalClose()
             {
                 if (!cursor.get())
                     return SQL_RESULT_SUCCESS;
diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp
index 2395d66864363..328bac958d987 100644
--- a/modules/platforms/cpp/odbc/src/statement.cpp
+++ b/modules/platforms/cpp/odbc/src/statement.cpp
@@ -16,6 +16,7 @@
  */
 
 #include "ignite/odbc/system/odbc_constants.h"
+#include "ignite/odbc/query/batch_query.h"
 #include "ignite/odbc/query/data_query.h"
 #include "ignite/odbc/query/column_metadata_query.h"
 #include "ignite/odbc/query/table_metadata_query.h"
@@ -39,9 +40,8 @@ namespace ignite
             currentQuery(),
             rowsFetched(0),
             rowStatuses(0),
-            paramBindOffset(0),
             columnBindOffset(0),
-            currentParamIdx(0)
+            parameters()
         {
             // No-op.
         }
@@ -91,8 +91,6 @@ namespace ignite
         void Statement::SafeBindColumn(uint16_t columnIdx, const app::ApplicationDataBuffer& buffer)
         {
             columnBindings[columnIdx] = buffer;
-
-            columnBindings[columnIdx].SetPtrToOffsetPtr(&columnBindOffset);
         }
 
         void Statement::SafeUnbindColumn(uint16_t columnIdx)
@@ -143,38 +141,45 @@ namespace ignite
         void Statement::BindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType, int16_t paramSqlType,
                                       SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen)
         {
-            IGNITE_ODBC_API_CALL(InternalBindParameter(paramIdx, ioType, bufferType, paramSqlType, columnSize, decDigits, buffer, bufferLen, resLen));
+            IGNITE_ODBC_API_CALL(InternalBindParameter(paramIdx, ioType, bufferType, paramSqlType, columnSize,
+                                                       decDigits, buffer, bufferLen, resLen));
         }
 
-        SqlResult Statement::InternalBindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType, int16_t paramSqlType,
-                                                   SqlUlen columnSize, int16_t decDigits, void* buffer, SqlLen bufferLen, SqlLen* resLen)
+
+        SqlResult Statement::InternalBindParameter(uint16_t paramIdx, int16_t ioType, int16_t bufferType,
+                                                   int16_t paramSqlType, SqlUlen columnSize, int16_t decDigits,
+                                                   void* buffer, SqlLen bufferLen, SqlLen* resLen)
         {
-            using namespace odbc::type_traits;
-            using odbc::Statement;
-            using odbc::app::ApplicationDataBuffer;
-            using odbc::app::Parameter;
-            using odbc::type_traits::IsSqlTypeSupported;
+            using namespace type_traits;
+            using app::ApplicationDataBuffer;
+            using app::Parameter;
 
             if (paramIdx == 0)
             {
-                AddStatusRecord(SQL_STATE_24000_INVALID_CURSOR_STATE,
-                    "The value specified for the argument ParameterNumber was less than 1.");
+                std::stringstream builder;
+                builder << "The value specified for the argument ParameterNumber was less than 1. [ParameterNumber=" << paramIdx << ']';
+
+                AddStatusRecord(SQL_STATE_24000_INVALID_CURSOR_STATE, builder.str());
 
                 return SQL_RESULT_ERROR;
             }
 
             if (ioType != SQL_PARAM_INPUT)
             {
-                AddStatusRecord(SQL_STATE_HY105_INVALID_PARAMETER_TYPE,
-                    "The value specified for the argument InputOutputType was not SQL_PARAM_INPUT.");
+                std::stringstream builder;
+                builder << "The value specified for the argument InputOutputType was not SQL_PARAM_INPUT. [ioType=" << ioType << ']';
+
+                AddStatusRecord(SQL_STATE_HY105_INVALID_PARAMETER_TYPE, builder.str());
 
                 return SQL_RESULT_ERROR;
             }
 
             if (!IsSqlTypeSupported(paramSqlType))
             {
-                AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED,
-                    "Data type is not supported.");
+                std::stringstream builder;
+                builder << "Data type is not supported. [typeId=" << paramSqlType << ']';
+
+                AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED, builder.str());
 
                 return SQL_RESULT_ERROR;
             }
@@ -183,8 +188,10 @@ namespace ignite
 
             if (driverType == IGNITE_ODBC_C_TYPE_UNSUPPORTED)
             {
-                AddStatusRecord(odbc::SQL_STATE_HY003_INVALID_APPLICATION_BUFFER_TYPE,
-                    "The argument TargetType was not a valid data type.");
+                std::stringstream builder;
+                builder << "The argument TargetType was not a valid data type. [TargetType=" << bufferType << ']';
+
+                AddStatusRecord(SQL_STATE_HY003_INVALID_APPLICATION_BUFFER_TYPE, builder.str());
 
                 return SQL_RESULT_ERROR;
             }
@@ -205,19 +212,17 @@ namespace ignite
 
         void Statement::SafeBindParameter(uint16_t paramIdx, const app::Parameter& param)
         {
-            paramBindings[paramIdx] = param;
-
-            paramBindings[paramIdx].GetBuffer().SetPtrToOffsetPtr(&paramBindOffset);
+            parameters.BindParameter(paramIdx, param);
         }
 
         void Statement::SafeUnbindParameter(uint16_t paramIdx)
         {
-            paramBindings.erase(paramIdx);
+            parameters.UnbindParameter(paramIdx);
         }
 
         void Statement::SafeUnbindAllParameters()
         {
-            paramBindings.clear();
+            parameters.UnbindAll();
         }
 
         void Statement::SetAttribute(int attr, void* value, SQLINTEGER valueLen)
@@ -231,7 +236,7 @@ namespace ignite
             {
                 case SQL_ATTR_ROW_ARRAY_SIZE:
                 {
-                    SQLULEN val = reinterpret_cast<SQLULEN>(value);
+                    SqlUlen val = reinterpret_cast<SqlUlen>(value);
 
                     LOG_MSG("SQL_ATTR_ROW_ARRAY_SIZE: " << val);
 
@@ -274,6 +279,20 @@ namespace ignite
                     break;
                 }
 
+                case SQL_ATTR_PARAMSET_SIZE:
+                {
+                    parameters.SetParamSetSize(reinterpret_cast<SqlUlen>(value));
+
+                    break;
+                }
+
+                case SQL_ATTR_PARAMS_PROCESSED_PTR:
+                {
+                    parameters.SetParamsProcessedPtr(reinterpret_cast<SqlUlen*>(value));
+
+                    break;
+                }
+
                 default:
                 {
                     AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED,
@@ -311,6 +330,9 @@ namespace ignite
 
                     *val = static_cast<SQLPOINTER>(this);
 
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
+
                     break;
                 }
 
@@ -320,14 +342,20 @@ namespace ignite
 
                     *val = static_cast<SQLINTEGER>(1);
 
+                    if (valueLen)
+                        *valueLen = SQL_IS_INTEGER;
+
                     break;
                 }
 
                 case SQL_ATTR_ROWS_FETCHED_PTR:
                 {
-                    SQLULEN** val = reinterpret_cast<SQLULEN**>(buf);
+                    SqlUlen** val = reinterpret_cast<SqlUlen**>(buf);
+
+                    *val = reinterpret_cast<SqlUlen*>(GetRowsFetchedPtr());
 
-                    *val = reinterpret_cast<SQLULEN*>(GetRowsFetchedPtr());
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
 
                     break;
                 }
@@ -338,6 +366,9 @@ namespace ignite
 
                     *val = reinterpret_cast<SQLUSMALLINT*>(GetRowStatusesPtr());
 
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
+
                     break;
                 }
 
@@ -345,16 +376,46 @@ namespace ignite
                 {
                     SQLULEN** val = reinterpret_cast<SQLULEN**>(buf);
 
-                    *val = reinterpret_cast<SQLULEN*>(GetParamBindOffsetPtr());
+                    *val = reinterpret_cast<SQLULEN*>(parameters.GetParamBindOffsetPtr());
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
 
                     break;
                 }
 
                 case SQL_ATTR_ROW_BIND_OFFSET_PTR:
                 {
-                    SQLULEN** val = reinterpret_cast<SQLULEN**>(buf);
+                    SqlUlen** val = reinterpret_cast<SqlUlen**>(buf);
 
-                    *val = reinterpret_cast<SQLULEN*>(GetColumnBindOffsetPtr());
+                    *val = reinterpret_cast<SqlUlen*>(GetColumnBindOffsetPtr());
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
+
+                    break;
+                }
+
+                case SQL_ATTR_PARAMSET_SIZE:
+                {
+                    SqlUlen* val = reinterpret_cast<SqlUlen*>(buf);
+
+                    *val = static_cast<SqlUlen>(parameters.GetParamSetSize());
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_UINTEGER;
+
+                    break;
+                }
+
+                case SQL_ATTR_PARAMS_PROCESSED_PTR:
+                {
+                    SqlUlen** val = reinterpret_cast<SqlUlen**>(buf);
+
+                    *val = parameters.GetParamsProcessedPtr();
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
 
                     break;
                 }
@@ -371,23 +432,45 @@ namespace ignite
             return SQL_RESULT_SUCCESS;
         }
 
-        uint16_t Statement::GetParametersNumber()
+        void Statement::GetParametersNumber(uint16_t& paramNum)
         {
-            IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
-
-            return static_cast<uint16_t>(paramBindings.size());
+            IGNITE_ODBC_API_CALL(InternalGetParametersNumber(paramNum));
         }
 
-        void Statement::SetParamBindOffsetPtr(int* ptr)
+        SqlResult Statement::InternalGetParametersNumber(uint16_t& paramNum)
         {
-            IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
+            if (!currentQuery.get())
+            {
+                AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR, "Query is not prepared.");
+
+                return SQL_RESULT_ERROR;
+            }
+
+            if (currentQuery->GetType() != query::Query::DATA)
+            {
+                paramNum = 0;
+
+                return SQL_RESULT_SUCCESS;
+            }
 
-            paramBindOffset = ptr;
+            if (!parameters.IsMetadataSet())
+            {
+                SqlResult res = UpdateParamsMeta();
+
+                if (res != SQL_RESULT_SUCCESS)
+                    return res;
+            }
+
+            paramNum = parameters.GetExpectedParamNum();
+
+            return SQL_RESULT_SUCCESS;
         }
 
-        int* Statement::GetParamBindOffsetPtr()
+        void Statement::SetParamBindOffsetPtr(int* ptr)
         {
-            return paramBindOffset;
+            IGNITE_ODBC_API_CALL_ALWAYS_SUCCESS;
+
+            parameters.SetParamBindOffsetPtr(ptr);
         }
 
         void Statement::GetColumnData(uint16_t columnIdx, app::ApplicationDataBuffer& buffer)
@@ -421,10 +504,10 @@ namespace ignite
             if (currentQuery.get())
                 currentQuery->Close();
 
-            currentQuery.reset(new query::DataQuery(*this, connection, query, paramBindings));
-
             // Resetting parameters types as we are changing the query.
-            paramTypes.clear();
+            parameters.Prepare();
+
+            currentQuery.reset(new query::DataQuery(*this, connection, query, parameters));
 
             return SQL_RESULT_SUCCESS;
         }
@@ -458,20 +541,31 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            bool paramDataReady = true;
-
-            app::ParameterBindingMap::iterator it;
-            for (it = paramBindings.begin(); it != paramBindings.end(); ++it)
+            if (parameters.GetParamSetSize() > 1 && currentQuery->GetType() == query::Query::DATA)
             {
-                app::Parameter& param = it->second;
+                query::DataQuery& qry = static_cast<query::DataQuery&>(*currentQuery);
 
-                param.ResetStoredData();
+                currentQuery.reset(new query::BatchQuery(*this, connection, qry.GetSql(), parameters));
+            }
+            else if (parameters.GetParamSetSize() == 1 && currentQuery->GetType() == query::Query::BATCH)
+            {
+                query::BatchQuery& qry = static_cast<query::BatchQuery&>(*currentQuery);
 
-                paramDataReady &= param.IsDataReady();
+                currentQuery.reset(new query::DataQuery(*this, connection, qry.GetSql(), parameters));
             }
 
-            if (!paramDataReady)
+            if (parameters.IsDataAtExecNeeded())
+            {
+                if (currentQuery->GetType() == query::Query::BATCH)
+                {
+                    AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED,
+                        "Data-at-execution is not supported together with batching.");
+
+                    return SQL_RESULT_ERROR;
+                }
+
                 return SQL_RESULT_NEED_DATA;
+            }
 
             return currentQuery->Execute();
         }
@@ -597,8 +691,10 @@ namespace ignite
         {
             if (!type_traits::IsSqlTypeSupported(sqlType))
             {
-                AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED,
-                    "Data type is not supported.");
+                std::stringstream builder;
+                builder << "Data type is not supported. [typeId=" << sqlType << ']';
+
+                AddStatusRecord(SQL_STATE_HYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED, builder.str());
 
                 return SQL_RESULT_ERROR;
             }
@@ -709,6 +805,12 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
+            if (columnBindOffset)
+            {
+                for (app::ColumnBindingMap::iterator it = columnBindings.begin(); it != columnBindings.end(); ++it)
+                    it->second.SetByteOffset(*columnBindOffset);
+            }
+
             SqlResult res = currentQuery->FetchNextRow(columnBindings);
 
             if (res == SQL_RESULT_SUCCESS)
@@ -877,35 +979,24 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            app::ParameterBindingMap::iterator it;
+            app::Parameter *selected = parameters.GetSelectedParameter();
 
-            if (currentParamIdx)
+            if (selected && !selected->IsDataReady())
             {
-                it = paramBindings.find(currentParamIdx);
+                AddStatusRecord(SQL_STATE_22026_DATA_LENGTH_MISMATCH,
+                    "Less data was sent for a parameter than was specified with "
+                    "the StrLen_or_IndPtr argument in SQLBindParameter.");
 
-                if (it != paramBindings.end() && !it->second.IsDataReady())
-                {
-                    AddStatusRecord(SQL_STATE_22026_DATA_LENGTH_MISMATCH,
-                        "Less data was sent for a parameter than was specified with "
-                        "the StrLen_or_IndPtr argument in SQLBindParameter.");
-
-                    return SQL_RESULT_ERROR;
-                }
+                return SQL_RESULT_ERROR;
             }
 
-            for (it = paramBindings.begin(); it != paramBindings.end(); ++it)
-            {
-                uint16_t paramIdx = it->first;
-                app::Parameter& param = it->second;
-
-                if (!param.IsDataReady())
-                {
-                    *paramPtr = param.GetBuffer().GetData();
+            selected = parameters.SelectNextParameter();
 
-                    currentParamIdx = paramIdx;
+            if (selected)
+            {
+                *paramPtr = selected->GetBuffer().GetData();
 
-                    return SQL_RESULT_NEED_DATA;
-                }
+                return SQL_RESULT_NEED_DATA;
             }
 
             SqlResult res = currentQuery->Execute();
@@ -932,7 +1023,7 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            if (currentParamIdx == 0)
+            if (!parameters.IsParameterSelected())
             {
                 AddStatusRecord(SQL_STATE_HY010_SEQUENCE_ERROR,
                     "Parameter is not selected with the SQLParamData.");
@@ -940,9 +1031,9 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            app::ParameterBindingMap::iterator it = paramBindings.find(currentParamIdx);
+            app::Parameter* param = parameters.GetSelectedParameter();
 
-            if (it == paramBindings.end())
+            if (!param)
             {
                 AddStatusRecord(SQL_STATE_HY000_GENERAL_ERROR,
                     "Selected parameter has been unbound.");
@@ -950,9 +1041,7 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            app::Parameter& param = it->second;
-
-            param.PutData(data, len);
+            param->PutData(data, len);
 
             return SQL_RESULT_SUCCESS;
         }
@@ -982,10 +1071,7 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            int8_t type = 0;
-
-            if (paramNum > 0 && static_cast<size_t>(paramNum) <= paramTypes.size())
-                type = paramTypes[paramNum - 1];
+            int8_t type = parameters.GetParamType(paramNum, 0);
 
             LOG_MSG("Type: " << type);
 
@@ -996,10 +1082,7 @@ namespace ignite
                 if (res != SQL_RESULT_SUCCESS)
                     return res;
 
-                if (paramNum < 1 || static_cast<size_t>(paramNum) > paramTypes.size())
-                    type = impl::binary::IGNITE_HDR_NULL;
-                else
-                    type = paramTypes[paramNum - 1];
+                type = parameters.GetParamType(paramNum, impl::binary::IGNITE_HDR_NULL);
             }
 
             if (dataType)
@@ -1052,11 +1135,11 @@ namespace ignite
                 return SQL_RESULT_ERROR;
             }
 
-            paramTypes = rsp.GetTypeIds();
+            parameters.UpdateParamsTypes(rsp.GetTypeIds());
 
-            for (size_t i = 0; i < paramTypes.size(); ++i)
+            for (size_t i = 0; i < rsp.GetTypeIds().size(); ++i)
             {
-                LOG_MSG("[" << i << "] Parameter type: " << paramTypes[i]);
+                LOG_MSG("[" << i << "] Parameter type: " << rsp.GetTypeIds()[i]);
             }
 
             return SQL_RESULT_SUCCESS;

From cbe5df51c423568782e31245c7f1aa06c9ba3be1 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 13 Jun 2017 19:47:00 +0300
Subject: [PATCH 235/516] IGNITE-5478: ODBC: SQLNumParams now returns number of
 required parameters.

(cherry picked from commit b1c56a1)

(cherry picked from commit 4a8f295)
---
 .../cpp/odbc-test/src/queries_test.cpp        | 30 +++++++++++++++++++
 .../cpp/odbc/include/ignite/odbc/statement.h  |  4 +--
 modules/platforms/cpp/odbc/src/odbc.cpp       |  7 ++---
 3 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 8e528b4c0b910..082def8ea0290 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -245,6 +245,25 @@ struct QueriesTestSuiteFixture
         BOOST_CHECK(ret == SQL_NO_DATA);
     }
 
+    void CheckParamsNum(const std::string& req, SQLSMALLINT expectedParamsNum)
+    {
+        std::vector<SQLCHAR> req0(req.begin(), req.end());
+
+        SQLRETURN ret = SQLPrepare(stmt, &req0[0], static_cast<SQLINTEGER>(req0.size()));
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        SQLSMALLINT paramsNum = -1;
+
+        ret = SQLNumParams(stmt, &paramsNum);
+
+        if (!SQL_SUCCEEDED(ret))
+            BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+        BOOST_CHECK_EQUAL(paramsNum, expectedParamsNum);
+    }
+
     int CountRows(SQLHSTMT stmt)
     {
         int res = 0;
@@ -1771,4 +1790,15 @@ BOOST_AUTO_TEST_CASE(TestTablesMeta)
     BOOST_CHECK(ret == SQL_NO_DATA);
 }
 
+BOOST_AUTO_TEST_CASE(TestParamsNum)
+{
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+
+    CheckParamsNum("SELECT * FROM TestType", 0);
+    CheckParamsNum("SELECT * FROM TestType WHERE _key=?", 1);
+    CheckParamsNum("SELECT * FROM TestType WHERE _key=? AND _val=?", 2);
+    CheckParamsNum("INSERT INTO TestType(_key, strField) VALUES(1, 'some')", 0);
+    CheckParamsNum("INSERT INTO TestType(_key, strField) VALUES(?, ?)", 2);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
index 132cf78e45238..93018cd5c41ac 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
@@ -129,9 +129,9 @@ namespace ignite
             void GetAttribute(int attr, void* buf, SQLINTEGER bufLen, SQLINTEGER *valueLen);
 
             /**
-             * Get number of binded parameters.
+             * Get number parameters required by the prepared statement.
              *
-             * @param paramNum Number of binded parameters.
+             * @param paramNum Number of parameters.
              */
             void GetParametersNumber(uint16_t& paramNum);
 
diff --git a/modules/platforms/cpp/odbc/src/odbc.cpp b/modules/platforms/cpp/odbc/src/odbc.cpp
index af8c73489902c..a85b694e9b3b8 100644
--- a/modules/platforms/cpp/odbc/src/odbc.cpp
+++ b/modules/platforms/cpp/odbc/src/odbc.cpp
@@ -898,11 +898,10 @@ namespace ignite
 
         if (paramCnt)
         {
-            uint16_t num = 0;
+            uint16_t paramNum = 0;
+            statement->GetParametersNumber(paramNum);
 
-            statement->GetParametersNumber(num);
-
-            *paramCnt = static_cast<SQLSMALLINT>(num);
+            *paramCnt = static_cast<SQLSMALLINT>(paramNum);
         }
 
         return statement->GetDiagnosticRecords().GetReturnCode();

From 9ad513e68b89e907f7db36a3f3f0daca0e5986e6 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Fri, 23 Jun 2017 17:06:40 +0300
Subject: [PATCH 236/516] IGNITE-5478: Fix for cherry pick

(cherry picked from commit a2a4ec1ee9794cb542f146a07c6c67002cad444e)
---
 modules/platforms/cpp/odbc-test/src/queries_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 082def8ea0290..ee470b3f7a114 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -1792,7 +1792,7 @@ BOOST_AUTO_TEST_CASE(TestTablesMeta)
 
 BOOST_AUTO_TEST_CASE(TestParamsNum)
 {
-    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
 
     CheckParamsNum("SELECT * FROM TestType", 0);
     CheckParamsNum("SELECT * FROM TestType WHERE _key=?", 1);

From bfec212b1ece0e9e791de6dfb642324834fa77ca Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Fri, 23 Jun 2017 00:24:57 +0300
Subject: [PATCH 237/516] Partially reverted GG-12352.

---
 .../spi/IgniteSpiOperationTimeoutHelper.java  |  4 +--
 .../tcp/TcpCommunicationSpi.java              |  1 -
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  6 +----
 .../apache/ignite/spi/GridTcpForwarder.java   | 26 +++++++++++++++++++
 4 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
index 1d9fa94f44475..33896361e2c21 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
@@ -96,9 +96,9 @@ public boolean checkFailureTimeoutReached(Exception e) {
         if (!failureDetectionTimeoutEnabled)
             return false;
 
-        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class))
+        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class, SocketException.class))
             return true;
 
-        return X.hasCause(e, SocketException.class) && (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
+        return (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 98183d8afbd7a..a87734e5c73e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2481,7 +2481,6 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
-                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 856370c2a0ee3..2f7e9b470aae6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1146,8 +1146,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
             boolean openSock = false;
 
-            boolean wasHandshake = false;
-
             Socket sock = null;
 
             try {
@@ -1165,8 +1163,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(
                     ackTimeout0));
 
-                wasHandshake = true;
-
                 if (msg instanceof TcpDiscoveryJoinRequestMessage) {
                     boolean ignore = false;
 
@@ -1268,7 +1264,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 if (!spi.failureDetectionTimeoutEnabled() && ++reconCnt == spi.getReconnectCount())
                     break;
 
-                if (!openSock || !wasHandshake) {
+                if (!openSock) {
                     // Reconnect for the second time, if connection is not established.
                     if (connectAttempts < 2) {
                         connectAttempts++;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
index d08321e18e8b5..68d97c1a3932b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
@@ -23,6 +23,7 @@
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -85,6 +86,10 @@ public GridTcpForwarder(
                                 outputCon.getInputStream(), inputCon.getOutputStream()
                             );
 
+                            //Force closing sibling if one of thread failed.
+                            forwardThread1.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread2));
+                            forwardThread2.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread1));
+
                             forwardThread1.start();
                             forwardThread2.start();
 
@@ -127,6 +132,25 @@ public GridTcpForwarder(
         U.join(mainThread, log);
     }
 
+    /**
+     *
+     */
+    private static class ForwarderExceptionHandler implements Thread.UncaughtExceptionHandler {
+        /** */
+        private Thread siblingThread;
+
+        /** */
+        public ForwarderExceptionHandler(Thread siblingThread) {
+
+            this.siblingThread = siblingThread;
+        }
+
+        /** */
+        @Override public void uncaughtException(Thread t, Throwable e) {
+            siblingThread.interrupt();
+        }
+    }
+
     /**
      * Thread reads data from input stream and write to output stream.
      */
@@ -166,6 +190,8 @@ private ForwardThread(String name, InputStream inputStream, OutputStream outputS
             }
             catch (IOException e) {
                 log.error("IOException while forwarding data [threadName=" + getName() + "]", e);
+
+                throw new IgniteException(e);
             }
         }
     }

From 1abc14fdc4e39d8245c3e50fb2cf3d183df08021 Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Fri, 23 Jun 2017 00:24:57 +0300
Subject: [PATCH 238/516] Partially reverted GG-12352.

---
 .../service/GridServiceProcessor.java         |  2 +-
 .../spi/IgniteSpiOperationTimeoutHelper.java  |  4 +--
 .../tcp/TcpCommunicationSpi.java              |  2 --
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  6 +----
 .../apache/ignite/spi/GridTcpForwarder.java   | 26 +++++++++++++++++++
 .../tcp/TcpCommunicationSpiDropNodesTest.java |  1 -
 .../TcpCommunicationSpiFaultyClientTest.java  |  5 ----
 7 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 009bf47c57101..20bcff7c780d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1657,7 +1657,7 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                                             log.info("Service processor detected a topology change during " +
                                                 "assignments calculation (will abort current iteration and " +
                                                 "re-calculate on the newer version): " +
-                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer + ']');
+                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer0 + ']');
 
                                         return;
                                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
index 1d9fa94f44475..33896361e2c21 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
@@ -96,9 +96,9 @@ public boolean checkFailureTimeoutReached(Exception e) {
         if (!failureDetectionTimeoutEnabled)
             return false;
 
-        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class))
+        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class, SocketException.class))
             return true;
 
-        return X.hasCause(e, SocketException.class) && (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
+        return (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 98183d8afbd7a..1bfa56accf2e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -23,7 +23,6 @@
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -2481,7 +2480,6 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
-                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 856370c2a0ee3..2f7e9b470aae6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1146,8 +1146,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
             boolean openSock = false;
 
-            boolean wasHandshake = false;
-
             Socket sock = null;
 
             try {
@@ -1165,8 +1163,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(
                     ackTimeout0));
 
-                wasHandshake = true;
-
                 if (msg instanceof TcpDiscoveryJoinRequestMessage) {
                     boolean ignore = false;
 
@@ -1268,7 +1264,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 if (!spi.failureDetectionTimeoutEnabled() && ++reconCnt == spi.getReconnectCount())
                     break;
 
-                if (!openSock || !wasHandshake) {
+                if (!openSock) {
                     // Reconnect for the second time, if connection is not established.
                     if (connectAttempts < 2) {
                         connectAttempts++;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
index d08321e18e8b5..68d97c1a3932b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
@@ -23,6 +23,7 @@
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -85,6 +86,10 @@ public GridTcpForwarder(
                                 outputCon.getInputStream(), inputCon.getOutputStream()
                             );
 
+                            //Force closing sibling if one of thread failed.
+                            forwardThread1.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread2));
+                            forwardThread2.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread1));
+
                             forwardThread1.start();
                             forwardThread2.start();
 
@@ -127,6 +132,25 @@ public GridTcpForwarder(
         U.join(mainThread, log);
     }
 
+    /**
+     *
+     */
+    private static class ForwarderExceptionHandler implements Thread.UncaughtExceptionHandler {
+        /** */
+        private Thread siblingThread;
+
+        /** */
+        public ForwarderExceptionHandler(Thread siblingThread) {
+
+            this.siblingThread = siblingThread;
+        }
+
+        /** */
+        @Override public void uncaughtException(Thread t, Throwable e) {
+            siblingThread.interrupt();
+        }
+    }
+
     /**
      * Thread reads data from input stream and write to output stream.
      */
@@ -166,6 +190,8 @@ private ForwardThread(String name, InputStream inputStream, OutputStream outputS
             }
             catch (IOException e) {
                 log.error("IOException while forwarding data [threadName=" + getName() + "]", e);
+
+                throw new IgniteException(e);
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
index 7b438d6795a8b..58a0dc170f9d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
@@ -42,7 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.config.GridTestProperties;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
index 33625e4b9edb4..dcfa91d6431a9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
@@ -18,16 +18,13 @@
 package org.apache.ignite.spi.communication.tcp;
 
 import java.io.IOException;
-import java.io.OutputStream;
 import java.net.InetAddress;
 import java.net.ServerSocket;
-import java.net.Socket;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
@@ -45,8 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 

From f59007f2c1221d8dd22abb0a9692b4abb31e87ad Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Fri, 23 Jun 2017 00:24:57 +0300
Subject: [PATCH 239/516] Partially reverted GG-12352.

---
 .../service/GridServiceProcessor.java         |  2 +-
 .../spi/IgniteSpiOperationTimeoutHelper.java  |  4 +--
 .../tcp/TcpCommunicationSpi.java              |  2 --
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 16 ++----------
 .../apache/ignite/spi/GridTcpForwarder.java   | 26 +++++++++++++++++++
 .../tcp/TcpCommunicationSpiDropNodesTest.java |  1 -
 .../TcpCommunicationSpiFaultyClientTest.java  |  5 ----
 .../discovery/tcp/TcpDiscoverySelfTest.java   |  1 +
 8 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 4c9f8e800465a..5c7a299b2bd4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1659,7 +1659,7 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                                             log.info("Service processor detected a topology change during " +
                                                 "assignments calculation (will abort current iteration and " +
                                                 "re-calculate on the newer version): " +
-                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer + ']');
+                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer0 + ']');
 
                                         return;
                                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
index 1d9fa94f44475..33896361e2c21 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
@@ -96,9 +96,9 @@ public boolean checkFailureTimeoutReached(Exception e) {
         if (!failureDetectionTimeoutEnabled)
             return false;
 
-        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class))
+        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class, SocketException.class))
             return true;
 
-        return X.hasCause(e, SocketException.class) && (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
+        return (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 269a98525cd42..10d2141ffdf15 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -23,7 +23,6 @@
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -2918,7 +2917,6 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
-                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 80a0f18359cdf..410a351586768 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1147,8 +1147,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
             boolean openSock = false;
 
-            boolean wasHandshake = false;
-
             Socket sock = null;
 
             try {
@@ -1166,8 +1164,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(
                     ackTimeout0));
 
-                wasHandshake = true;
-
                 if (msg instanceof TcpDiscoveryJoinRequestMessage) {
                     boolean ignore = false;
 
@@ -1247,7 +1243,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
                 errs.add(e);
 
-                if (X.hasCause(e, SSLException.class)) {
+                if (X.hasCause(e, SSLException.class, StreamCorruptedException.class)) {
                     if (--sslConnectAttempts == 0)
                         throw new IgniteException("Unable to establish secure connection. " +
                             "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
@@ -1255,21 +1251,13 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                     continue;
                 }
 
-                if (X.hasCause(e, StreamCorruptedException.class)) {
-                    if (--sslConnectAttempts == 0)
-                        throw new IgniteException("Unable to establish plain connection. " +
-                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
-
-                    continue;
-                }
-
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
                 if (!spi.failureDetectionTimeoutEnabled() && ++reconCnt == spi.getReconnectCount())
                     break;
 
-                if (!openSock || !wasHandshake) {
+                if (!openSock) {
                     // Reconnect for the second time, if connection is not established.
                     if (connectAttempts < 2) {
                         connectAttempts++;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
index d08321e18e8b5..68d97c1a3932b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
@@ -23,6 +23,7 @@
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -85,6 +86,10 @@ public GridTcpForwarder(
                                 outputCon.getInputStream(), inputCon.getOutputStream()
                             );
 
+                            //Force closing sibling if one of thread failed.
+                            forwardThread1.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread2));
+                            forwardThread2.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread1));
+
                             forwardThread1.start();
                             forwardThread2.start();
 
@@ -127,6 +132,25 @@ public GridTcpForwarder(
         U.join(mainThread, log);
     }
 
+    /**
+     *
+     */
+    private static class ForwarderExceptionHandler implements Thread.UncaughtExceptionHandler {
+        /** */
+        private Thread siblingThread;
+
+        /** */
+        public ForwarderExceptionHandler(Thread siblingThread) {
+
+            this.siblingThread = siblingThread;
+        }
+
+        /** */
+        @Override public void uncaughtException(Thread t, Throwable e) {
+            siblingThread.interrupt();
+        }
+    }
+
     /**
      * Thread reads data from input stream and write to output stream.
      */
@@ -166,6 +190,8 @@ private ForwardThread(String name, InputStream inputStream, OutputStream outputS
             }
             catch (IOException e) {
                 log.error("IOException while forwarding data [threadName=" + getName() + "]", e);
+
+                throw new IgniteException(e);
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
index 71a758024b6a8..3315c17046af5 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
@@ -42,7 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.config.GridTestProperties;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
index c18a89f04e041..b0e543df495c1 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
@@ -18,16 +18,13 @@
 package org.apache.ignite.spi.communication.tcp;
 
 import java.io.IOException;
-import java.io.OutputStream;
 import java.net.InetAddress;
 import java.net.ServerSocket;
-import java.net.Socket;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
@@ -45,8 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
index 043208c955b44..cd1776ca2bc1c 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
@@ -219,6 +219,7 @@ else if (gridName.contains("testNodeShutdownOnRingMessageWorkerFailureFailedNode
         return cfg;
     }
 
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         discoMap = null;

From 3e2a8dd7497cc21f9ad176213bcefa35869eb198 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 26 Jun 2017 12:27:58 +0300
Subject: [PATCH 240/516] Minor fix for GG-12197 "Ignore events for discarded
 update in CLOCK mode".

---
 .../processors/cache/GridCacheMapEntry.java       | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 6ae2a7cc3a77a..bfa20d9bed139 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -2280,13 +2280,14 @@ else if (ttl != CU.TTL_ZERO)
                             if (updateCntr != null)
                                 updateCntr0 = updateCntr;
 
-                            cctx.continuousQueries().skipUpdateEvent(
-                                lsnrs,
-                                key,
-                                partition(),
-                                updateCntr0,
-                                primary,
-                                topVer);
+                            if (lsnrs != null)
+                                cctx.continuousQueries().skipUpdateEvent(
+                                    lsnrs,
+                                    key,
+                                    partition(),
+                                    updateCntr0,
+                                    primary,
+                                    topVer);
                         }
 
                         return new GridCacheUpdateAtomicResult(false,

From 03e507f5021f3aee9728e5cbcbc85ed731e5e538 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 26 Jun 2017 19:37:27 +0300
Subject: [PATCH 241/516] Fixed tests.

---
 .../ignite/internal/IgniteClientReconnectAbstractTest.java   | 5 +++++
 ...iteTxCacheWriteSynchronizationModesMultithreadedTest.java | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
index 4d49366e1ad23..a793760f44fbf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
@@ -69,6 +69,9 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
     /** */
     private static final long RECONNECT_TIMEOUT = 10_000;
 
+    /** Reconnect should occurs before failure detection time is out. */
+    public static final long FAILURE_DETECTION_TIMEOUT = RECONNECT_TIMEOUT +2000;
+
     /** */
     protected boolean clientMode;
 
@@ -76,6 +79,8 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
+        cfg.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT);
+
         TestTcpDiscoverySpi disco = new TestTcpDiscoverySpi();
 
         disco.setIpFinder(ipFinder);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
index 08396daa5efeb..c214a772d422c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cache.store.CacheStoreAdapter;
@@ -104,6 +105,8 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+
         startGrids(SRVS);
 
         clientMode = true;
@@ -119,6 +122,8 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         super.afterTestsStopped();
     }
 

From ad6add47b29caf44dadfe2e32acd4d245ee256ab Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 27 Jun 2017 10:34:42 +0300
Subject: [PATCH 242/516] Fixed tests.

---
 ...IgniteBinaryMetadataUpdateNodeRestartTest.java | 10 ++++++++++
 .../IgniteCacheNearRestartRollbackSelfTest.java   | 15 +++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
index 814fb08eacba5..3d552847fd71d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheEntryProcessor;
 import org.apache.ignite.cluster.ClusterTopologyException;
@@ -88,10 +89,19 @@ public class IgniteBinaryMetadataUpdateNodeRestartTest extends GridCommonAbstrac
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         super.afterTestsStopped();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index a48497d8f39c4..3f242b5726867 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -31,6 +31,7 @@
 import javax.cache.processor.MutableEntry;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -92,6 +93,20 @@ public class IgniteCacheNearRestartRollbackSelfTest extends GridCommonAbstractTe
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+    }
+
     /**
      * @param gridName Grid name.
      * @return Cache configuration.

From 897f4c00c4945eda3f9f4a41d064ded2f6f27ccc Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 27 Jun 2017 12:55:11 +0300
Subject: [PATCH 243/516] gg-12133 Deadlock for metadata update from
 GridCacheMapEntry.innerUpdate

---
 .../checkpoint/GridCheckpointManager.java     |   2 +-
 .../managers/communication/GridIoManager.java |  10 +-
 .../communication/GridMessageListener.java    |   3 +-
 .../GridDeploymentCommunication.java          |   4 +-
 .../eventstorage/GridEventStorageManager.java |   4 +-
 .../processors/cache/GridCacheIoManager.java  |  37 ++--
 .../cache/transactions/IgniteTxManager.java   |   2 +-
 .../clock/GridClockSyncProcessor.java         |   2 +-
 .../continuous/GridContinuousProcessor.java   |   4 +-
 .../datastreamer/DataStreamProcessor.java     |   2 +-
 .../datastreamer/DataStreamerImpl.java        |   2 +-
 .../processors/igfs/IgfsDataManager.java      |   2 +-
 .../igfs/IgfsFragmentizerManager.java         |   4 +-
 .../processors/job/GridJobProcessor.java      |   8 +-
 .../handlers/task/GridTaskCommandHandler.java |   4 +-
 .../processors/task/GridTaskProcessor.java    |   6 +-
 .../jobstealing/JobStealingCollisionSpi.java  |   2 +-
 .../TestRecordingCommunicationSpi.java        |  29 +++
 ...CommunicationManagerListenersSelfTest.java |   2 +-
 .../GridCommunicationSendMessageSelfTest.java |   2 +-
 .../GridCachePartitionedGetSelfTest.java      |   2 +-
 .../IgniteBinaryMetadataUpdateFromInvoke.java | 187 ++++++++++++++++++
 .../communication/GridIoManagerBenchmark.java |   4 +-
 .../GridIoManagerBenchmark0.java              |  12 +-
 .../GridCacheMessageSelfTest.java             |   2 +-
 .../testframework/GridSpiTestContext.java     |   4 +-
 .../IgniteCacheRestartTestSuite2.java         |   2 +
 .../query/h2/opt/GridH2IndexBase.java         |   2 +-
 .../h2/twostep/GridMapQueryExecutor.java      |   2 +-
 .../h2/twostep/GridReduceQueryExecutor.java   |   2 +-
 30 files changed, 289 insertions(+), 61 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
index 9124cafa79a6b..40c2c636ace93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
@@ -449,7 +449,7 @@ private class CheckpointRequestListener implements GridMessageListener {
          * @param msg Received message.
          */
         @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridCheckpointRequest req = (GridCheckpointRequest)msg;
 
             if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 4b68e5bdd0da7..a4e32980114e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -321,7 +321,7 @@ public void resetMetrics() {
             log.debug(startInfo());
 
         addMessageListener(TOPIC_IO_TEST, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 ClusterNode node = ctx.discovery().node(nodeId);
 
                 if (node == null)
@@ -1508,7 +1508,7 @@ private void invokeListener(Byte plc, GridMessageListener lsnr, UUID nodeId, Obj
             CUR_PLC.set(plc);
 
         try {
-            lsnr.onMessage(nodeId, msg);
+            lsnr.onMessage(nodeId, msg, plc);
         }
         finally {
             if (change)
@@ -2333,14 +2333,14 @@ private static class ArrayListener implements GridMessageListener {
          * @param nodeId Node ID.
          * @param msg Message.
          */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridMessageListener[] arr0 = arr;
 
             if (arr0 == null)
                 return;
 
             for (GridMessageListener l : arr0)
-                l.onMessage(nodeId, msg);
+                l.onMessage(nodeId, msg, plc);
         }
 
         /**
@@ -2440,7 +2440,7 @@ private class GridUserMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
         @SuppressWarnings({"SynchronizationOnLocalVariableOrMethodParameter", "ConstantConditions",
             "OverlyStrongTypeCast"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (!(msg instanceof GridIoUserMessage)) {
                 U.error(log, "Received unknown message (potentially fatal problem): " + msg);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
index 39935917c3e65..c7de57c959c18 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
@@ -30,6 +30,7 @@ public interface GridMessageListener extends EventListener {
      * @param nodeId ID of node that sent the message. Note that may have already
      *      left topology by the time this message is received.
      * @param msg Message received.
+     * @param plc Message policy (pool).
      */
-    public void onMessage(UUID nodeId, Object msg);
+    public void onMessage(UUID nodeId, Object msg, byte plc);
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
index a571ae445438f..661bf68b61a42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
@@ -82,7 +82,7 @@ class GridDeploymentCommunication {
         this.log = log.getLogger(getClass());
 
         peerLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 processDeploymentRequest(nodeId, msg);
             }
         };
@@ -416,7 +416,7 @@ GridDeploymentResponse sendResourceRequest(final String rsrcName, IgniteUuid cls
         };
 
         GridMessageListener resLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert nodeId != null;
                 assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
index 607bb9688aaf1..b77ec27496c50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
@@ -877,7 +877,7 @@ private <T extends Event> List<T> query(IgnitePredicate<T> p, Collection<? exten
 
         GridMessageListener resLsnr = new GridMessageListener() {
             @SuppressWarnings("deprecation")
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert nodeId != null;
                 assert msg != null;
 
@@ -1054,7 +1054,7 @@ private boolean[] copy(boolean[] arr) {
      */
     private class RequestListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 924ce7917b4eb..39087b628beb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -127,7 +128,7 @@ public class GridCacheIoManager extends GridCacheSharedManagerAdapter {
 
     /** Message listener. */
     private GridMessageListener lsnr = new GridMessageListener() {
-        @Override public void onMessage(final UUID nodeId, final Object msg) {
+        @Override public void onMessage(final UUID nodeId, final Object msg, final byte plc) {
             if (log.isDebugEnabled())
                 log.debug("Received unordered cache communication message [nodeId=" + nodeId +
                     ", locId=" + cctx.localNodeId() + ", msg=" + msg + ']');
@@ -206,6 +207,9 @@ else if (cacheDesc.receivedFromStartVersion() != null)
                         log.debug(msg0.toString());
                     }
 
+                    if (plc == GridIoPolicy.UTILITY_CACHE_POOL)
+                        rmtAffVer = new AffinityTopologyVersion(cctx.localNode().order());
+
                     fut = cctx.exchange().affinityReadyFuture(rmtAffVer);
                 }
             }
@@ -213,22 +217,27 @@ else if (cacheDesc.receivedFromStartVersion() != null)
             if (fut != null && !fut.isDone()) {
                 fut.listen(new CI1<IgniteInternalFuture<?>>() {
                     @Override public void apply(IgniteInternalFuture<?> t) {
-                        cctx.kernalContext().closure().runLocalSafe(new Runnable() {
-                            @Override public void run() {
-                                IgniteLogger log = cacheMsg.messageLogger(cctx);
+                        try {
+                            cctx.kernalContext().pools().poolForPolicy(plc).execute(new Runnable() {
+                                @Override public void run() {
+                                    IgniteLogger log = cacheMsg.messageLogger(cctx);
 
-                                if (log.isDebugEnabled()) {
-                                    StringBuilder msg0 = new StringBuilder("Process cache message after wait for " +
-                                        "affinity topology version [");
+                                    if (log.isDebugEnabled()) {
+                                        StringBuilder msg0 = new StringBuilder("Process cache message after wait for " +
+                                            "affinity topology version [");
 
-                                    appendMessageInfo(cacheMsg, nodeId, msg0).append(']');
+                                        appendMessageInfo(cacheMsg, nodeId, msg0).append(']');
 
-                                    log.debug(msg0.toString());
-                                }
+                                        log.debug(msg0.toString());
+                                    }
 
-                                handleMessage(nodeId, cacheMsg);
-                            }
-                        });
+                                    handleMessage(nodeId, cacheMsg);
+                                }
+                            });
+                        }
+                        catch (IgniteCheckedException e) {
+                            U.error(cacheMsg.messageLogger(cctx), "Failed to get pool for policy: " + plc, e);
+                        }
                     }
                 });
 
@@ -1336,7 +1345,7 @@ private class OrderedMessageListener implements GridMessageListener {
 
         /** {@inheritDoc} */
         @SuppressWarnings({"CatchGenericClass", "unchecked"})
-        @Override public void onMessage(final UUID nodeId, Object msg) {
+        @Override public void onMessage(final UUID nodeId, Object msg, byte plc) {
             if (log.isDebugEnabled())
                 log.debug("Received cache ordered message [nodeId=" + nodeId + ", msg=" + msg + ']');
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 2c02f967d00be..2f21614352e8c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -2472,7 +2472,7 @@ private CommitListener(IgniteInternalTx tx) {
     private class DeadlockDetectionListener implements GridMessageListener {
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridCacheMessage cacheMsg = (GridCacheMessage)msg;
 
             unmarshall(nodeId, cacheMsg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
index 07643164368ce..3586956da99dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
@@ -96,7 +96,7 @@ public GridClockSyncProcessor(GridKernalContext ctx) {
         srv.start(ctx);
 
         ctx.io().addMessageListener(TOPIC_TIME_SYNC, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert msg instanceof GridClockDeltaSnapshotMessage;
 
                 GridClockDeltaSnapshotMessage msg0 = (GridClockDeltaSnapshotMessage)msg;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 9fd9b6d44b89e..f0429bc3a3c91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -288,7 +288,7 @@ public GridContinuousProcessor(GridKernalContext ctx) {
             });
 
         ctx.io().addMessageListener(TOPIC_CONTINUOUS, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object obj) {
+            @Override public void onMessage(UUID nodeId, Object obj, byte plc) {
                 GridContinuousMessage msg = (GridContinuousMessage)obj;
 
                 if (msg.data() == null && msg.dataBytes() != null) {
@@ -721,7 +721,7 @@ public IgniteInternalFuture<UUID> startRoutine(GridContinuousHandler hnd,
     private void registerMessageListener(GridContinuousHandler hnd) {
         if (hnd.orderedTopic() != null) {
             ctx.io().addMessageListener(hnd.orderedTopic(), new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object obj) {
+                @Override public void onMessage(UUID nodeId, Object obj, byte plc) {
                     GridContinuousMessage msg = (GridContinuousMessage)obj;
 
                     // Only notification can be ordered.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index fee4dd6616b28..6f35a52d5b673 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -80,7 +80,7 @@ public DataStreamProcessor(GridKernalContext ctx) {
 
         if (!ctx.clientNode()) {
             ctx.io().addMessageListener(TOPIC_DATASTREAM, new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     assert msg instanceof DataStreamerRequest;
 
                     processRequest(nodeId, (DataStreamerRequest)msg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index bb9ffdd5795a2..515314eb08728 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -321,7 +321,7 @@ public DataStreamerImpl(
         topic = TOPIC_DATASTREAM.topic(IgniteUuid.fromUuid(ctx.localNodeId()));
 
         ctx.io().addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert msg instanceof DataStreamerResponse;
 
                 DataStreamerResponse res = (DataStreamerResponse)msg;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index e534800efd7a6..59e1b724764e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -153,7 +153,7 @@ void awaitInit() {
         topic = F.isEmpty(igfsName) ? TOPIC_IGFS : TOPIC_IGFS.topic(igfsName);
 
         igfsCtx.kernalContext().io().addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (msg instanceof IgfsBlocksMessage)
                     processBlocksMessage(nodeId, (IgfsBlocksMessage)msg);
                 else if (msg instanceof IgfsAckMessage)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
index 2e82f33024b57..f76b877f38443 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
@@ -453,7 +453,7 @@ protected FragmentizerCoordinator() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof IgfsFragmentizerResponse) {
                 IgfsFragmentizerResponse res = (IgfsFragmentizerResponse)msg;
 
@@ -673,7 +673,7 @@ protected FragmentizerWorker() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof IgfsFragmentizerRequest ||
                 msg instanceof IgfsSyncMessage) {
                 if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index ea9cbd7dbd272..eb3300ce3412c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -444,7 +444,7 @@ public Collection<ComputeJobSibling> requestJobSiblings(
         final Condition cond = lock.newCondition();
 
         GridMessageListener msgLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 String err = null;
                 GridJobSiblingsResponse res = null;
 
@@ -1842,7 +1842,7 @@ private class JobHoldListener implements GridJobHoldListener {
      */
     private class JobSessionListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
@@ -1858,7 +1858,7 @@ private class JobSessionListener implements GridMessageListener {
      */
     private class JobCancelListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
@@ -1876,7 +1876,7 @@ private class JobCancelListener implements GridMessageListener {
      */
     private class JobExecutionListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
index 947435cb58d83..88f8d4de9241e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
@@ -109,7 +109,7 @@ public GridTaskCommandHandler(final GridKernalContext ctx) {
         super(ctx);
 
         ctx.io().addMessageListener(TOPIC_REST, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!(msg instanceof GridTaskResultRequest)) {
                     U.warn(log, "Received unexpected message instead of task result request: " + msg);
 
@@ -425,7 +425,7 @@ private IgniteBiTuple<String, GridTaskResultResponse> requestTaskResult(final UU
         final Condition cond = lock.newCondition();
 
         GridMessageListener msgLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 String err = null;
                 GridTaskResultResponse res = null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 12213581b3286..a807ad9efe5dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -1225,7 +1225,7 @@ private JobMessageListener(boolean jobResOnly) {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof GridJobExecuteResponse)
                 processJobExecuteResponse(nodeId, (GridJobExecuteResponse)msg);
             else if (jobResOnly)
@@ -1269,7 +1269,7 @@ private class TaskDiscoveryListener implements GridLocalEventListener {
      */
     private class JobSiblingsMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (!(msg instanceof GridJobSiblingsRequest)) {
                 U.warn(log, "Received unexpected message instead of siblings request: " + msg);
 
@@ -1341,7 +1341,7 @@ private class JobSiblingsMessageListener implements GridMessageListener {
      */
     private class TaskCancelMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert msg != null;
 
             if (!(msg instanceof GridTaskCancelRequest)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
index f778bfcc73ae8..fca249893980a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
@@ -547,7 +547,7 @@ public void setStealingAttributes(Map<String, ? extends Serializable> stealAttrs
 
         spiCtx.addMessageListener(
             msgLsnr = new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     MessageInfo info = rcvMsgMap.get(nodeId);
 
                     if (info == null) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
index 2aed459fedc59..17ca1a7ffc7c4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
@@ -88,6 +88,8 @@ public class TestRecordingCommunicationSpi extends TcpCommunicationSpi {
 
                     blockedMsgs.add(new T2<>(node, ioMsg));
 
+                    this.notifyAll();
+
                     return;
                 }
             }
@@ -136,6 +138,33 @@ public boolean hasBlockedMessages() {
         }
     }
 
+    /**
+     * @param cls Message class.
+     * @param nodeName Node name.
+     * @throws InterruptedException If interrupted.
+     */
+    public void waitForBlocked(Class<?> cls, String nodeName) throws InterruptedException {
+        synchronized (this) {
+            while (!hasMessage(cls, nodeName))
+                wait();
+        }
+    }
+
+    /**
+     * @param cls Message class.
+     * @param nodeName Node name.
+     * @return {@code True} if has blocked message.
+     */
+    private boolean hasMessage(Class<?> cls, String nodeName) {
+        for (T2<ClusterNode, GridIoMessage> msg : blockedMsgs) {
+            if (msg.get2().message().getClass() == cls &&
+                nodeName.equals(msg.get1().attribute(ATTR_GRID_NAME)))
+                return true;
+        }
+
+        return false;
+    }
+
     /**
      * @param blockP Message block predicate.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
index 7613543335ce5..9289f866c8fb5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
@@ -159,7 +159,7 @@ private static class MessageListeningTask extends ComputeTaskSplitAdapter<Object
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             // No-op.
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
index 8503b48f5ec48..08ee3475e9f9a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
@@ -121,7 +121,7 @@ private void doSend() throws Exception {
         final CountDownLatch latch = new CountDownLatch(SAMPLE_CNT);
 
         mgr1.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 latch.countDown();
             }
         });
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
index 2cb4a005676b7..47d6092f2b2c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
@@ -226,7 +226,7 @@ private void prepare() throws Exception {
                 ((IgniteKernal)g).context().io().addMessageListener(
                     TOPIC_CACHE,
                     new GridMessageListener() {
-                        @Override public void onMessage(UUID nodeId, Object msg) {
+                        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                             info("Received message from node [nodeId=" + nodeId + ", msg=" + msg + ']');
 
                             if (msg instanceof GridNearSingleGetRequest) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java
new file mode 100644
index 0000000000000..f919ca736115c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class IgniteBinaryMetadataUpdateFromInvoke extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setMarshaller(null);
+
+        cfg.setClientMode(client);
+
+        CacheConfiguration ccfg = new CacheConfiguration("cache");
+
+        ccfg.setAtomicityMode(ATOMIC);
+        ccfg.setBackups(1);
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        cfg.setConsistentId(gridName);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMetadataUpdateFromAtomicInvoke() throws Exception {
+        for (int i = 0; i < 2; i++) {
+            client = false;
+
+            log.info("Iteration: " + i);
+
+            final int SRVS = 4;
+
+            startGrids(SRVS);
+
+            awaitPartitionMapExchange();
+
+            final CyclicBarrier b = new CyclicBarrier(3);
+
+            final AtomicBoolean stop = new AtomicBoolean();
+
+            final int META_PRIMARY_NODE = 0;
+            final int META_UPDATE_FROM_NODE = 1;
+
+            BinaryMarshaller marsh = (BinaryMarshaller)ignite(META_PRIMARY_NODE).configuration().getMarshaller();
+
+            marsh.getContext().registerClass(
+                marsh.binaryMarshaller().context().typeId(TestEnum1.class.getName()), TestEnum1.class);
+
+            TestRecordingCommunicationSpi testSpi =
+                (TestRecordingCommunicationSpi)ignite(META_UPDATE_FROM_NODE).configuration().getCommunicationSpi();
+
+            testSpi.blockMessages(GridNearTxPrepareRequest.class, getTestGridName(META_PRIMARY_NODE));
+
+            IgniteInternalFuture<?> sFut1 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    IgniteCache<Object, Object> cache = ignite(META_UPDATE_FROM_NODE).cache("cache");
+
+                    List<Integer> keys = primaryKeys(cache, 1);
+
+                    b.await();
+
+                    cache.invoke(keys.get(0), new TestEntryProcessor());
+
+                    return null;
+                }
+            }, "async-node-0");
+
+            IgniteInternalFuture<?> sFut2 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    IgniteCache<Object, Object> cache = ignite(META_PRIMARY_NODE).cache("cache");
+
+                    List<Integer> keys = primaryKeys(cache, 1);
+
+                    b.await();
+
+                    Thread.sleep(2000);
+
+                    cache.invoke(keys.get(0), new TestEntryProcessor());
+
+                    return null;
+                }
+            }, "async-node-1");
+
+            IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    client = true;
+
+                    b.await();
+
+                    for (int i = 0; i < 1; i++)
+                        startGrid(SRVS + i);
+
+                    stop.set(true);
+
+                    return null;
+                }
+            });
+
+            testSpi.waitForBlocked(GridNearTxPrepareRequest.class, getTestGridName(META_PRIMARY_NODE));
+
+            U.sleep(5000);
+
+            testSpi.stopBlock();
+
+            sFut1.get();
+            sFut2.get();
+            fut2.get();
+
+            stopAllGrids();
+        }
+    }
+
+    /**
+     *
+     */
+    static class TestEntryProcessor implements CacheEntryProcessor<Object, Object, Object> {
+        @Override public Object process(MutableEntry<Object, Object> e, Object... args) {
+            e.setValue(TestEnum1.ENUM);
+
+            return null;
+        }
+    }
+
+    /**
+     *
+     */
+    enum TestEnum1 {
+        /** */
+        ENUM
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
index 723495cfa89c2..293e5789ddf3c 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
@@ -240,7 +240,7 @@ private static void receiveMessages(final IgniteKernal g) {
         GridMessageListener lsnr = new GridMessageListener() {
             private ClusterNode node;
 
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (node == null)
                     node = g.context().discovery().node(nodeId);
 
@@ -336,7 +336,7 @@ private ClusterNode awaitOther(final GridDiscoveryManager disc) throws Interrupt
      */
     private static class SenderMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             msgCntr.increment();
 
             if (testLatency)
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
index f2c62559397b4..83efd7de42098 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
@@ -130,7 +130,7 @@ public void testThroughput() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -141,7 +141,7 @@ public void testThroughput() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 msgCntr.increment();
 
                 sem.release();
@@ -224,7 +224,7 @@ public void testLatency() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -235,7 +235,7 @@ public void testLatency() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 map.get(((GridTestMessage)msg).id()).countDown();
             }
         });
@@ -324,7 +324,7 @@ public void testVariableLoad() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -335,7 +335,7 @@ public void testVariableLoad() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 msgCntr.increment();
 
                 sem.release();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
index 9c975422a2601..2d04db2fedab9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
@@ -139,7 +139,7 @@ private void doSend() throws Exception {
         final CountDownLatch latch = new CountDownLatch(SAMPLE_CNT);
 
         mgr1.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 try {
                     latch.countDown();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index 1c8acbc3960ce..0c04039753beb 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -328,7 +328,7 @@ public Serializable removeSentMessage(ClusterNode node) {
     @SuppressWarnings("deprecation")
     public void triggerMessage(ClusterNode node, Object msg) {
         for (GridMessageListener lsnr : msgLsnrs)
-            lsnr.onMessage(node.id(), msg);
+            lsnr.onMessage(node.id(), msg, (byte)0);
     }
 
     /** {@inheritDoc} */
@@ -667,7 +667,7 @@ private class GridLocalMessageListener implements GridMessageListener {
         @SuppressWarnings({
             "SynchronizationOnLocalVariableOrMethodParameter", "ConstantConditions",
             "OverlyStrongTypeCast"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridIoUserMessage ioMsg = (GridIoUserMessage)msg;
 
             ClusterNode node = locNode;
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
index 05137868443c8..fb38c5535b35b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.GridCachePutAllFailoverSelfTest;
+import org.apache.ignite.internal.processors.cache.IgniteBinaryMetadataUpdateFromInvoke;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicPutAllFailoverSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCachePutAllRestartTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteBinaryMetadataUpdateNodeRestartTest;
@@ -45,6 +46,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCachePutAllFailoverSelfTest.class);
 
         suite.addTestSuite(IgniteBinaryMetadataUpdateNodeRestartTest.class);
+        suite.addTestSuite(IgniteBinaryMetadataUpdateFromInvoke.class);
 
         suite.addTestSuite(IgniteCacheGetRestartTest.class);
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index c29239f143719..22b94c7ac796b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -138,7 +138,7 @@ protected final void initDistributedJoinMessaging(GridH2Table tbl) {
             msgTopic = new IgniteBiTuple<>(GridTopic.TOPIC_QUERY, tbl.identifier() + '.' + getName());
 
             msgLsnr = new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     GridSpinBusyLock l = desc.indexing().busyLock();
 
                     if (!l.enterBusy())
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index ac1a6a60c0200..0605287ded597 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -167,7 +167,7 @@ public void start(final GridKernalContext ctx, IgniteH2Indexing h2) throws Ignit
         }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
 
         ctx.io().addMessageListener(GridTopic.TOPIC_QUERY, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!busyLock.enterBusy())
                     return;
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 3f886ee686d66..71c0e1245b625 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -190,7 +190,7 @@ public void start(final GridKernalContext ctx, final IgniteH2Indexing h2) throws
         log = ctx.log(GridReduceQueryExecutor.class);
 
         ctx.io().addMessageListener(GridTopic.TOPIC_QUERY, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!busyLock.enterBusy())
                     return;
 

From 90e96db22dbb7a341bfe1a8130b6dc16d5d8ae81 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Tue, 27 Jun 2017 14:43:06 +0300
Subject: [PATCH 244/516] Compilation is fixed.

---
 .../java/org/apache/ignite/IgniteSystemProperties.java    | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index f473c51e74a93..e9bbf5a10df4c 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -320,6 +320,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_H2_DEBUG_CONSOLE = "IGNITE_H2_DEBUG_CONSOLE";
 
+    /**
+     * This property allows to specify user defined port which H2 indexing SPI will use
+     * to start H2 debug console on. If this property is not set or set to 0, H2 debug
+     * console will use system-provided dynamic port.
+     * This property is only relevant when {@link #IGNITE_H2_DEBUG_CONSOLE} property is set.
+     */
+    public static final String IGNITE_H2_DEBUG_CONSOLE_PORT = "IGNITE_H2_DEBUG_CONSOLE_PORT";
+
     /**
      * If this property is set to {@code true} then shared memory space native debug will be enabled.
      */

From 3f33a902ed0d0a3e27be548209fe8e7933da57a9 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 26 Jun 2017 12:27:58 +0300
Subject: [PATCH 245/516] Minor fix for GG-12197 "Ignore events for discarded
 update in CLOCK mode".

---
 .../processors/cache/GridCacheMapEntry.java       | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 6ae2a7cc3a77a..bfa20d9bed139 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -2280,13 +2280,14 @@ else if (ttl != CU.TTL_ZERO)
                             if (updateCntr != null)
                                 updateCntr0 = updateCntr;
 
-                            cctx.continuousQueries().skipUpdateEvent(
-                                lsnrs,
-                                key,
-                                partition(),
-                                updateCntr0,
-                                primary,
-                                topVer);
+                            if (lsnrs != null)
+                                cctx.continuousQueries().skipUpdateEvent(
+                                    lsnrs,
+                                    key,
+                                    partition(),
+                                    updateCntr0,
+                                    primary,
+                                    topVer);
                         }
 
                         return new GridCacheUpdateAtomicResult(false,

From 3668b91817c1da7958cc3d7c4dddf890a2237772 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 27 Jun 2017 10:34:42 +0300
Subject: [PATCH 246/516] Partially reverted GG-12352.

---
 .../service/GridServiceProcessor.java         |  2 +-
 .../spi/IgniteSpiOperationTimeoutHelper.java  |  4 +--
 .../tcp/TcpCommunicationSpi.java              |  2 --
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 16 ++----------
 .../IgniteClientReconnectAbstractTest.java    |  5 ++++
 ...teBinaryMetadataUpdateNodeRestartTest.java | 10 +++++++
 ...gniteCacheNearRestartRollbackSelfTest.java | 15 +++++++++++
 ...SynchronizationModesMultithreadedTest.java |  5 ++++
 .../apache/ignite/spi/GridTcpForwarder.java   | 26 +++++++++++++++++++
 .../tcp/TcpCommunicationSpiDropNodesTest.java |  1 -
 .../TcpCommunicationSpiFaultyClientTest.java  |  5 ----
 .../discovery/tcp/TcpDiscoverySelfTest.java   |  1 +
 12 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 4c9f8e800465a..5c7a299b2bd4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1659,7 +1659,7 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                                             log.info("Service processor detected a topology change during " +
                                                 "assignments calculation (will abort current iteration and " +
                                                 "re-calculate on the newer version): " +
-                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer + ']');
+                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer0 + ']');
 
                                         return;
                                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
index 1d9fa94f44475..33896361e2c21 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiOperationTimeoutHelper.java
@@ -96,9 +96,9 @@ public boolean checkFailureTimeoutReached(Exception e) {
         if (!failureDetectionTimeoutEnabled)
             return false;
 
-        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class))
+        if (X.hasCause(e, IgniteSpiOperationTimeoutException.class, SocketTimeoutException.class, SocketException.class))
             return true;
 
-        return X.hasCause(e, SocketException.class) && (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
+        return (timeout - (U.currentTimeMillis() - lastOperStartTs) <= 0);
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 269a98525cd42..10d2141ffdf15 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -23,7 +23,6 @@
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -2918,7 +2917,6 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
-                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 80a0f18359cdf..410a351586768 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1147,8 +1147,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
             boolean openSock = false;
 
-            boolean wasHandshake = false;
-
             Socket sock = null;
 
             try {
@@ -1166,8 +1164,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                 TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(
                     ackTimeout0));
 
-                wasHandshake = true;
-
                 if (msg instanceof TcpDiscoveryJoinRequestMessage) {
                     boolean ignore = false;
 
@@ -1247,7 +1243,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
                 errs.add(e);
 
-                if (X.hasCause(e, SSLException.class)) {
+                if (X.hasCause(e, SSLException.class, StreamCorruptedException.class)) {
                     if (--sslConnectAttempts == 0)
                         throw new IgniteException("Unable to establish secure connection. " +
                             "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
@@ -1255,21 +1251,13 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                     continue;
                 }
 
-                if (X.hasCause(e, StreamCorruptedException.class)) {
-                    if (--sslConnectAttempts == 0)
-                        throw new IgniteException("Unable to establish plain connection. " +
-                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
-
-                    continue;
-                }
-
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
                 if (!spi.failureDetectionTimeoutEnabled() && ++reconCnt == spi.getReconnectCount())
                     break;
 
-                if (!openSock || !wasHandshake) {
+                if (!openSock) {
                     // Reconnect for the second time, if connection is not established.
                     if (connectAttempts < 2) {
                         connectAttempts++;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
index 4d49366e1ad23..a793760f44fbf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
@@ -69,6 +69,9 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
     /** */
     private static final long RECONNECT_TIMEOUT = 10_000;
 
+    /** Reconnect should occurs before failure detection time is out. */
+    public static final long FAILURE_DETECTION_TIMEOUT = RECONNECT_TIMEOUT +2000;
+
     /** */
     protected boolean clientMode;
 
@@ -76,6 +79,8 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
+        cfg.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT);
+
         TestTcpDiscoverySpi disco = new TestTcpDiscoverySpi();
 
         disco.setIpFinder(ipFinder);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
index 814fb08eacba5..3d552847fd71d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheEntryProcessor;
 import org.apache.ignite.cluster.ClusterTopologyException;
@@ -88,10 +89,19 @@ public class IgniteBinaryMetadataUpdateNodeRestartTest extends GridCommonAbstrac
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         super.afterTestsStopped();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index a48497d8f39c4..3f242b5726867 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -31,6 +31,7 @@
 import javax.cache.processor.MutableEntry;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -92,6 +93,20 @@ public class IgniteCacheNearRestartRollbackSelfTest extends GridCommonAbstractTe
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+    }
+
     /**
      * @param gridName Grid name.
      * @return Cache configuration.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
index 08396daa5efeb..c214a772d422c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cache.store.CacheStoreAdapter;
@@ -104,6 +105,8 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+
         startGrids(SRVS);
 
         clientMode = true;
@@ -119,6 +122,8 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         super.afterTestsStopped();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
index d08321e18e8b5..68d97c1a3932b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpForwarder.java
@@ -23,6 +23,7 @@
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -85,6 +86,10 @@ public GridTcpForwarder(
                                 outputCon.getInputStream(), inputCon.getOutputStream()
                             );
 
+                            //Force closing sibling if one of thread failed.
+                            forwardThread1.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread2));
+                            forwardThread2.setUncaughtExceptionHandler(new ForwarderExceptionHandler(forwardThread1));
+
                             forwardThread1.start();
                             forwardThread2.start();
 
@@ -127,6 +132,25 @@ public GridTcpForwarder(
         U.join(mainThread, log);
     }
 
+    /**
+     *
+     */
+    private static class ForwarderExceptionHandler implements Thread.UncaughtExceptionHandler {
+        /** */
+        private Thread siblingThread;
+
+        /** */
+        public ForwarderExceptionHandler(Thread siblingThread) {
+
+            this.siblingThread = siblingThread;
+        }
+
+        /** */
+        @Override public void uncaughtException(Thread t, Throwable e) {
+            siblingThread.interrupt();
+        }
+    }
+
     /**
      * Thread reads data from input stream and write to output stream.
      */
@@ -166,6 +190,8 @@ private ForwardThread(String name, InputStream inputStream, OutputStream outputS
             }
             catch (IOException e) {
                 log.error("IOException while forwarding data [threadName=" + getName() + "]", e);
+
+                throw new IgniteException(e);
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
index 71a758024b6a8..3315c17046af5 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiDropNodesTest.java
@@ -42,7 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.config.GridTestProperties;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
index c18a89f04e041..b0e543df495c1 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiFaultyClientTest.java
@@ -18,16 +18,13 @@
 package org.apache.ignite.spi.communication.tcp;
 
 import java.io.IOException;
-import java.io.OutputStream;
 import java.net.InetAddress;
 import java.net.ServerSocket;
-import java.net.Socket;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
@@ -45,8 +42,6 @@
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
-import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
index 043208c955b44..cd1776ca2bc1c 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
@@ -219,6 +219,7 @@ else if (gridName.contains("testNodeShutdownOnRingMessageWorkerFailureFailedNode
         return cfg;
     }
 
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         discoMap = null;

From 172f41c489c0ca5b7613163cd325b0c01d5b28b1 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 27 Jun 2017 12:55:11 +0300
Subject: [PATCH 247/516] gg-12133 Deadlock for metadata update from
 GridCacheMapEntry.innerUpdate

(cherry picked from commit 897f4c0)
---
 .../checkpoint/GridCheckpointManager.java     |   2 +-
 .../managers/communication/GridIoManager.java |  10 +-
 .../communication/GridMessageListener.java    |   3 +-
 .../GridDeploymentCommunication.java          |   4 +-
 .../eventstorage/GridEventStorageManager.java |   4 +-
 .../processors/cache/GridCacheIoManager.java  |  37 ++--
 .../cache/transactions/IgniteTxManager.java   |   2 +-
 .../clock/GridClockSyncProcessor.java         |   2 +-
 .../continuous/GridContinuousProcessor.java   |   4 +-
 .../datastreamer/DataStreamProcessor.java     |   2 +-
 .../datastreamer/DataStreamerImpl.java        |   2 +-
 .../processors/igfs/IgfsDataManager.java      |   2 +-
 .../igfs/IgfsFragmentizerManager.java         |   4 +-
 .../processors/job/GridJobProcessor.java      |   8 +-
 .../handlers/task/GridTaskCommandHandler.java |   4 +-
 .../processors/task/GridTaskProcessor.java    |   6 +-
 .../jobstealing/JobStealingCollisionSpi.java  |   2 +-
 .../TestRecordingCommunicationSpi.java        |  29 +++
 ...CommunicationManagerListenersSelfTest.java |   2 +-
 .../GridCommunicationSendMessageSelfTest.java |   2 +-
 .../GridCachePartitionedGetSelfTest.java      |   2 +-
 .../IgniteBinaryMetadataUpdateFromInvoke.java | 187 ++++++++++++++++++
 ...adlockDetectionMessageMarshallingTest.java |   2 +-
 .../communication/GridIoManagerBenchmark.java |   4 +-
 .../GridIoManagerBenchmark0.java              |  12 +-
 .../GridCacheMessageSelfTest.java             |   2 +-
 .../testframework/GridSpiTestContext.java     |   4 +-
 .../IgniteCacheRestartTestSuite2.java         |   2 +
 .../query/h2/opt/GridH2IndexBase.java         |   2 +-
 .../h2/twostep/GridMapQueryExecutor.java      |   2 +-
 .../h2/twostep/GridReduceQueryExecutor.java   |   2 +-
 31 files changed, 290 insertions(+), 62 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
index 9124cafa79a6b..40c2c636ace93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
@@ -449,7 +449,7 @@ private class CheckpointRequestListener implements GridMessageListener {
          * @param msg Received message.
          */
         @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridCheckpointRequest req = (GridCheckpointRequest)msg;
 
             if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 06dcb8664445c..2cb4c3eebe1a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -322,7 +322,7 @@ public void resetMetrics() {
             log.debug(startInfo());
 
         addMessageListener(TOPIC_IO_TEST, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 ClusterNode node = ctx.discovery().node(nodeId);
 
                 if (node == null)
@@ -1509,7 +1509,7 @@ private void invokeListener(Byte plc, GridMessageListener lsnr, UUID nodeId, Obj
             CUR_PLC.set(plc);
 
         try {
-            lsnr.onMessage(nodeId, msg);
+            lsnr.onMessage(nodeId, msg, plc);
         }
         finally {
             if (change)
@@ -2334,14 +2334,14 @@ private static class ArrayListener implements GridMessageListener {
          * @param nodeId Node ID.
          * @param msg Message.
          */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridMessageListener[] arr0 = arr;
 
             if (arr0 == null)
                 return;
 
             for (GridMessageListener l : arr0)
-                l.onMessage(nodeId, msg);
+                l.onMessage(nodeId, msg, plc);
         }
 
         /**
@@ -2441,7 +2441,7 @@ private class GridUserMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
         @SuppressWarnings({"SynchronizationOnLocalVariableOrMethodParameter", "ConstantConditions",
             "OverlyStrongTypeCast"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (!(msg instanceof GridIoUserMessage)) {
                 U.error(log, "Received unknown message (potentially fatal problem): " + msg);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
index 39935917c3e65..c7de57c959c18 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridMessageListener.java
@@ -30,6 +30,7 @@ public interface GridMessageListener extends EventListener {
      * @param nodeId ID of node that sent the message. Note that may have already
      *      left topology by the time this message is received.
      * @param msg Message received.
+     * @param plc Message policy (pool).
      */
-    public void onMessage(UUID nodeId, Object msg);
+    public void onMessage(UUID nodeId, Object msg, byte plc);
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
index a571ae445438f..661bf68b61a42 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java
@@ -82,7 +82,7 @@ class GridDeploymentCommunication {
         this.log = log.getLogger(getClass());
 
         peerLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 processDeploymentRequest(nodeId, msg);
             }
         };
@@ -416,7 +416,7 @@ GridDeploymentResponse sendResourceRequest(final String rsrcName, IgniteUuid cls
         };
 
         GridMessageListener resLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert nodeId != null;
                 assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
index 406da2d284b63..a80cfbe2555d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
@@ -977,7 +977,7 @@ private <T extends Event> List<T> query(IgnitePredicate<T> p, Collection<? exten
 
         GridMessageListener resLsnr = new GridMessageListener() {
             @SuppressWarnings("deprecation")
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert nodeId != null;
                 assert msg != null;
 
@@ -1154,7 +1154,7 @@ private boolean[] copy(boolean[] arr) {
      */
     private class RequestListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 924ce7917b4eb..39087b628beb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -127,7 +128,7 @@ public class GridCacheIoManager extends GridCacheSharedManagerAdapter {
 
     /** Message listener. */
     private GridMessageListener lsnr = new GridMessageListener() {
-        @Override public void onMessage(final UUID nodeId, final Object msg) {
+        @Override public void onMessage(final UUID nodeId, final Object msg, final byte plc) {
             if (log.isDebugEnabled())
                 log.debug("Received unordered cache communication message [nodeId=" + nodeId +
                     ", locId=" + cctx.localNodeId() + ", msg=" + msg + ']');
@@ -206,6 +207,9 @@ else if (cacheDesc.receivedFromStartVersion() != null)
                         log.debug(msg0.toString());
                     }
 
+                    if (plc == GridIoPolicy.UTILITY_CACHE_POOL)
+                        rmtAffVer = new AffinityTopologyVersion(cctx.localNode().order());
+
                     fut = cctx.exchange().affinityReadyFuture(rmtAffVer);
                 }
             }
@@ -213,22 +217,27 @@ else if (cacheDesc.receivedFromStartVersion() != null)
             if (fut != null && !fut.isDone()) {
                 fut.listen(new CI1<IgniteInternalFuture<?>>() {
                     @Override public void apply(IgniteInternalFuture<?> t) {
-                        cctx.kernalContext().closure().runLocalSafe(new Runnable() {
-                            @Override public void run() {
-                                IgniteLogger log = cacheMsg.messageLogger(cctx);
+                        try {
+                            cctx.kernalContext().pools().poolForPolicy(plc).execute(new Runnable() {
+                                @Override public void run() {
+                                    IgniteLogger log = cacheMsg.messageLogger(cctx);
 
-                                if (log.isDebugEnabled()) {
-                                    StringBuilder msg0 = new StringBuilder("Process cache message after wait for " +
-                                        "affinity topology version [");
+                                    if (log.isDebugEnabled()) {
+                                        StringBuilder msg0 = new StringBuilder("Process cache message after wait for " +
+                                            "affinity topology version [");
 
-                                    appendMessageInfo(cacheMsg, nodeId, msg0).append(']');
+                                        appendMessageInfo(cacheMsg, nodeId, msg0).append(']');
 
-                                    log.debug(msg0.toString());
-                                }
+                                        log.debug(msg0.toString());
+                                    }
 
-                                handleMessage(nodeId, cacheMsg);
-                            }
-                        });
+                                    handleMessage(nodeId, cacheMsg);
+                                }
+                            });
+                        }
+                        catch (IgniteCheckedException e) {
+                            U.error(cacheMsg.messageLogger(cctx), "Failed to get pool for policy: " + plc, e);
+                        }
                     }
                 });
 
@@ -1336,7 +1345,7 @@ private class OrderedMessageListener implements GridMessageListener {
 
         /** {@inheritDoc} */
         @SuppressWarnings({"CatchGenericClass", "unchecked"})
-        @Override public void onMessage(final UUID nodeId, Object msg) {
+        @Override public void onMessage(final UUID nodeId, Object msg, byte plc) {
             if (log.isDebugEnabled())
                 log.debug("Received cache ordered message [nodeId=" + nodeId + ", msg=" + msg + ']');
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 1ba0102408eb0..8b029465fbf30 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -2475,7 +2475,7 @@ private CommitListener(IgniteInternalTx tx) {
     private class DeadlockDetectionListener implements GridMessageListener {
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridCacheMessage cacheMsg = (GridCacheMessage)msg;
 
             Throwable err = null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
index 07643164368ce..3586956da99dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
@@ -96,7 +96,7 @@ public GridClockSyncProcessor(GridKernalContext ctx) {
         srv.start(ctx);
 
         ctx.io().addMessageListener(TOPIC_TIME_SYNC, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert msg instanceof GridClockDeltaSnapshotMessage;
 
                 GridClockDeltaSnapshotMessage msg0 = (GridClockDeltaSnapshotMessage)msg;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 4717e8724d038..c4989a218df34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -288,7 +288,7 @@ public GridContinuousProcessor(GridKernalContext ctx) {
             });
 
         ctx.io().addMessageListener(TOPIC_CONTINUOUS, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object obj) {
+            @Override public void onMessage(UUID nodeId, Object obj, byte plc) {
                 GridContinuousMessage msg = (GridContinuousMessage)obj;
 
                 if (msg.data() == null && msg.dataBytes() != null) {
@@ -722,7 +722,7 @@ public IgniteInternalFuture<UUID> startRoutine(GridContinuousHandler hnd,
     private void registerMessageListener(GridContinuousHandler hnd) {
         if (hnd.orderedTopic() != null) {
             ctx.io().addMessageListener(hnd.orderedTopic(), new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object obj) {
+                @Override public void onMessage(UUID nodeId, Object obj, byte plc) {
                     GridContinuousMessage msg = (GridContinuousMessage)obj;
 
                     // Only notification can be ordered.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index fee4dd6616b28..6f35a52d5b673 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -80,7 +80,7 @@ public DataStreamProcessor(GridKernalContext ctx) {
 
         if (!ctx.clientNode()) {
             ctx.io().addMessageListener(TOPIC_DATASTREAM, new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     assert msg instanceof DataStreamerRequest;
 
                     processRequest(nodeId, (DataStreamerRequest)msg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index bb9ffdd5795a2..515314eb08728 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -321,7 +321,7 @@ public DataStreamerImpl(
         topic = TOPIC_DATASTREAM.topic(IgniteUuid.fromUuid(ctx.localNodeId()));
 
         ctx.io().addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 assert msg instanceof DataStreamerResponse;
 
                 DataStreamerResponse res = (DataStreamerResponse)msg;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index 4490a6854d79b..21a0b7f2037d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -155,7 +155,7 @@ void awaitInit() {
         topic = F.isEmpty(igfsName) ? TOPIC_IGFS : TOPIC_IGFS.topic(igfsName);
 
         igfsCtx.kernalContext().io().addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (msg instanceof IgfsBlocksMessage)
                     processBlocksMessage(nodeId, (IgfsBlocksMessage)msg);
                 else if (msg instanceof IgfsAckMessage)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
index 2e82f33024b57..f76b877f38443 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerManager.java
@@ -453,7 +453,7 @@ protected FragmentizerCoordinator() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof IgfsFragmentizerResponse) {
                 IgfsFragmentizerResponse res = (IgfsFragmentizerResponse)msg;
 
@@ -673,7 +673,7 @@ protected FragmentizerWorker() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof IgfsFragmentizerRequest ||
                 msg instanceof IgfsSyncMessage) {
                 if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index ea9cbd7dbd272..eb3300ce3412c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -444,7 +444,7 @@ public Collection<ComputeJobSibling> requestJobSiblings(
         final Condition cond = lock.newCondition();
 
         GridMessageListener msgLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 String err = null;
                 GridJobSiblingsResponse res = null;
 
@@ -1842,7 +1842,7 @@ private class JobHoldListener implements GridJobHoldListener {
      */
     private class JobSessionListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
@@ -1858,7 +1858,7 @@ private class JobSessionListener implements GridMessageListener {
      */
     private class JobCancelListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
@@ -1876,7 +1876,7 @@ private class JobCancelListener implements GridMessageListener {
      */
     private class JobExecutionListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert nodeId != null;
             assert msg != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
index 947435cb58d83..88f8d4de9241e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskCommandHandler.java
@@ -109,7 +109,7 @@ public GridTaskCommandHandler(final GridKernalContext ctx) {
         super(ctx);
 
         ctx.io().addMessageListener(TOPIC_REST, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!(msg instanceof GridTaskResultRequest)) {
                     U.warn(log, "Received unexpected message instead of task result request: " + msg);
 
@@ -425,7 +425,7 @@ private IgniteBiTuple<String, GridTaskResultResponse> requestTaskResult(final UU
         final Condition cond = lock.newCondition();
 
         GridMessageListener msgLsnr = new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 String err = null;
                 GridTaskResultResponse res = null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 12213581b3286..a807ad9efe5dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -1225,7 +1225,7 @@ private JobMessageListener(boolean jobResOnly) {
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (msg instanceof GridJobExecuteResponse)
                 processJobExecuteResponse(nodeId, (GridJobExecuteResponse)msg);
             else if (jobResOnly)
@@ -1269,7 +1269,7 @@ private class TaskDiscoveryListener implements GridLocalEventListener {
      */
     private class JobSiblingsMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             if (!(msg instanceof GridJobSiblingsRequest)) {
                 U.warn(log, "Received unexpected message instead of siblings request: " + msg);
 
@@ -1341,7 +1341,7 @@ private class JobSiblingsMessageListener implements GridMessageListener {
      */
     private class TaskCancelMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             assert msg != null;
 
             if (!(msg instanceof GridTaskCancelRequest)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
index f778bfcc73ae8..fca249893980a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingCollisionSpi.java
@@ -547,7 +547,7 @@ public void setStealingAttributes(Map<String, ? extends Serializable> stealAttrs
 
         spiCtx.addMessageListener(
             msgLsnr = new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     MessageInfo info = rcvMsgMap.get(nodeId);
 
                     if (info == null) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
index 2aed459fedc59..17ca1a7ffc7c4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
@@ -88,6 +88,8 @@ public class TestRecordingCommunicationSpi extends TcpCommunicationSpi {
 
                     blockedMsgs.add(new T2<>(node, ioMsg));
 
+                    this.notifyAll();
+
                     return;
                 }
             }
@@ -136,6 +138,33 @@ public boolean hasBlockedMessages() {
         }
     }
 
+    /**
+     * @param cls Message class.
+     * @param nodeName Node name.
+     * @throws InterruptedException If interrupted.
+     */
+    public void waitForBlocked(Class<?> cls, String nodeName) throws InterruptedException {
+        synchronized (this) {
+            while (!hasMessage(cls, nodeName))
+                wait();
+        }
+    }
+
+    /**
+     * @param cls Message class.
+     * @param nodeName Node name.
+     * @return {@code True} if has blocked message.
+     */
+    private boolean hasMessage(Class<?> cls, String nodeName) {
+        for (T2<ClusterNode, GridIoMessage> msg : blockedMsgs) {
+            if (msg.get2().message().getClass() == cls &&
+                nodeName.equals(msg.get1().attribute(ATTR_GRID_NAME)))
+                return true;
+        }
+
+        return false;
+    }
+
     /**
      * @param blockP Message block predicate.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
index 7613543335ce5..9289f866c8fb5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationManagerListenersSelfTest.java
@@ -159,7 +159,7 @@ private static class MessageListeningTask extends ComputeTaskSplitAdapter<Object
         }
 
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             // No-op.
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
index 8503b48f5ec48..08ee3475e9f9a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/GridCommunicationSendMessageSelfTest.java
@@ -121,7 +121,7 @@ private void doSend() throws Exception {
         final CountDownLatch latch = new CountDownLatch(SAMPLE_CNT);
 
         mgr1.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 latch.countDown();
             }
         });
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
index 2cb4a005676b7..47d6092f2b2c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java
@@ -226,7 +226,7 @@ private void prepare() throws Exception {
                 ((IgniteKernal)g).context().io().addMessageListener(
                     TOPIC_CACHE,
                     new GridMessageListener() {
-                        @Override public void onMessage(UUID nodeId, Object msg) {
+                        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                             info("Received message from node [nodeId=" + nodeId + ", msg=" + msg + ']');
 
                             if (msg instanceof GridNearSingleGetRequest) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java
new file mode 100644
index 0000000000000..f919ca736115c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryMetadataUpdateFromInvoke.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class IgniteBinaryMetadataUpdateFromInvoke extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setMarshaller(null);
+
+        cfg.setClientMode(client);
+
+        CacheConfiguration ccfg = new CacheConfiguration("cache");
+
+        ccfg.setAtomicityMode(ATOMIC);
+        ccfg.setBackups(1);
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        cfg.setConsistentId(gridName);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMetadataUpdateFromAtomicInvoke() throws Exception {
+        for (int i = 0; i < 2; i++) {
+            client = false;
+
+            log.info("Iteration: " + i);
+
+            final int SRVS = 4;
+
+            startGrids(SRVS);
+
+            awaitPartitionMapExchange();
+
+            final CyclicBarrier b = new CyclicBarrier(3);
+
+            final AtomicBoolean stop = new AtomicBoolean();
+
+            final int META_PRIMARY_NODE = 0;
+            final int META_UPDATE_FROM_NODE = 1;
+
+            BinaryMarshaller marsh = (BinaryMarshaller)ignite(META_PRIMARY_NODE).configuration().getMarshaller();
+
+            marsh.getContext().registerClass(
+                marsh.binaryMarshaller().context().typeId(TestEnum1.class.getName()), TestEnum1.class);
+
+            TestRecordingCommunicationSpi testSpi =
+                (TestRecordingCommunicationSpi)ignite(META_UPDATE_FROM_NODE).configuration().getCommunicationSpi();
+
+            testSpi.blockMessages(GridNearTxPrepareRequest.class, getTestGridName(META_PRIMARY_NODE));
+
+            IgniteInternalFuture<?> sFut1 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    IgniteCache<Object, Object> cache = ignite(META_UPDATE_FROM_NODE).cache("cache");
+
+                    List<Integer> keys = primaryKeys(cache, 1);
+
+                    b.await();
+
+                    cache.invoke(keys.get(0), new TestEntryProcessor());
+
+                    return null;
+                }
+            }, "async-node-0");
+
+            IgniteInternalFuture<?> sFut2 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    IgniteCache<Object, Object> cache = ignite(META_PRIMARY_NODE).cache("cache");
+
+                    List<Integer> keys = primaryKeys(cache, 1);
+
+                    b.await();
+
+                    Thread.sleep(2000);
+
+                    cache.invoke(keys.get(0), new TestEntryProcessor());
+
+                    return null;
+                }
+            }, "async-node-1");
+
+            IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    client = true;
+
+                    b.await();
+
+                    for (int i = 0; i < 1; i++)
+                        startGrid(SRVS + i);
+
+                    stop.set(true);
+
+                    return null;
+                }
+            });
+
+            testSpi.waitForBlocked(GridNearTxPrepareRequest.class, getTestGridName(META_PRIMARY_NODE));
+
+            U.sleep(5000);
+
+            testSpi.stopBlock();
+
+            sFut1.get();
+            sFut2.get();
+            fut2.get();
+
+            stopAllGrids();
+        }
+    }
+
+    /**
+     *
+     */
+    static class TestEntryProcessor implements CacheEntryProcessor<Object, Object, Object> {
+        @Override public Object process(MutableEntry<Object, Object> e, Object... args) {
+            e.setValue(TestEnum1.ENUM);
+
+            return null;
+        }
+    }
+
+    /**
+     *
+     */
+    enum TestEnum1 {
+        /** */
+        ENUM
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
index eafd09f2fdbf9..f32148a272cbf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionMessageMarshallingTest.java
@@ -76,7 +76,7 @@ public void testMessageUnmarshallWithoutCacheContext() throws Exception {
             final AtomicBoolean res = new AtomicBoolean();
 
             clientCtx.gridIO().addMessageListener(TOPIC, new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     if (msg instanceof TxLocksResponse) {
                         try {
                             ((TxLocksResponse)msg).finishUnmarshal(clientCtx, clientCtx.deploy().globalLoader());
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
index 723495cfa89c2..293e5789ddf3c 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark.java
@@ -240,7 +240,7 @@ private static void receiveMessages(final IgniteKernal g) {
         GridMessageListener lsnr = new GridMessageListener() {
             private ClusterNode node;
 
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (node == null)
                     node = g.context().discovery().node(nodeId);
 
@@ -336,7 +336,7 @@ private ClusterNode awaitOther(final GridDiscoveryManager disc) throws Interrupt
      */
     private static class SenderMessageListener implements GridMessageListener {
         /** {@inheritDoc} */
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             msgCntr.increment();
 
             if (testLatency)
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
index f2c62559397b4..83efd7de42098 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/communication/GridIoManagerBenchmark0.java
@@ -130,7 +130,7 @@ public void testThroughput() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -141,7 +141,7 @@ public void testThroughput() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 msgCntr.increment();
 
                 sem.release();
@@ -224,7 +224,7 @@ public void testLatency() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -235,7 +235,7 @@ public void testLatency() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 map.get(((GridTestMessage)msg).id()).countDown();
             }
         });
@@ -324,7 +324,7 @@ public void testVariableLoad() throws Exception {
         rcv.addMessageListener(
             topic,
             new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     try {
                         rcv.send(sndNode, topic, (Message)msg, PUBLIC_POOL);
                     }
@@ -335,7 +335,7 @@ public void testVariableLoad() throws Exception {
             });
 
         snd.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 msgCntr.increment();
 
                 sem.release();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
index 9c975422a2601..2d04db2fedab9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
@@ -139,7 +139,7 @@ private void doSend() throws Exception {
         final CountDownLatch latch = new CountDownLatch(SAMPLE_CNT);
 
         mgr1.addMessageListener(topic, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 try {
                     latch.countDown();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index 1c8acbc3960ce..0c04039753beb 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -328,7 +328,7 @@ public Serializable removeSentMessage(ClusterNode node) {
     @SuppressWarnings("deprecation")
     public void triggerMessage(ClusterNode node, Object msg) {
         for (GridMessageListener lsnr : msgLsnrs)
-            lsnr.onMessage(node.id(), msg);
+            lsnr.onMessage(node.id(), msg, (byte)0);
     }
 
     /** {@inheritDoc} */
@@ -667,7 +667,7 @@ private class GridLocalMessageListener implements GridMessageListener {
         @SuppressWarnings({
             "SynchronizationOnLocalVariableOrMethodParameter", "ConstantConditions",
             "OverlyStrongTypeCast"})
-        @Override public void onMessage(UUID nodeId, Object msg) {
+        @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
             GridIoUserMessage ioMsg = (GridIoUserMessage)msg;
 
             ClusterNode node = locNode;
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
index 05137868443c8..fb38c5535b35b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.GridCachePutAllFailoverSelfTest;
+import org.apache.ignite.internal.processors.cache.IgniteBinaryMetadataUpdateFromInvoke;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicPutAllFailoverSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCachePutAllRestartTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteBinaryMetadataUpdateNodeRestartTest;
@@ -45,6 +46,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCachePutAllFailoverSelfTest.class);
 
         suite.addTestSuite(IgniteBinaryMetadataUpdateNodeRestartTest.class);
+        suite.addTestSuite(IgniteBinaryMetadataUpdateFromInvoke.class);
 
         suite.addTestSuite(IgniteCacheGetRestartTest.class);
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index c29239f143719..22b94c7ac796b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -138,7 +138,7 @@ protected final void initDistributedJoinMessaging(GridH2Table tbl) {
             msgTopic = new IgniteBiTuple<>(GridTopic.TOPIC_QUERY, tbl.identifier() + '.' + getName());
 
             msgLsnr = new GridMessageListener() {
-                @Override public void onMessage(UUID nodeId, Object msg) {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                     GridSpinBusyLock l = desc.indexing().busyLock();
 
                     if (!l.enterBusy())
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index ac1a6a60c0200..0605287ded597 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -167,7 +167,7 @@ public void start(final GridKernalContext ctx, IgniteH2Indexing h2) throws Ignit
         }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
 
         ctx.io().addMessageListener(GridTopic.TOPIC_QUERY, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!busyLock.enterBusy())
                     return;
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index ee9976c22846a..cbfe1de8735aa 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -189,7 +189,7 @@ public void start(final GridKernalContext ctx, final IgniteH2Indexing h2) throws
         log = ctx.log(GridReduceQueryExecutor.class);
 
         ctx.io().addMessageListener(GridTopic.TOPIC_QUERY, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 if (!busyLock.enterBusy())
                     return;
 

From 0521b7780756788d92bfa35ef00f56b5bb01367d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 27 Jun 2017 20:43:07 +0300
Subject: [PATCH 248/516] GG-12370: Fixed massive NODE_FAILED events lead to
 excessive momentary memory consumption.

---
 .../GridCachePartitionExchangeManager.java    |  5 +-
 .../CacheLateAffinityAssignmentTest.java      | 68 +++++++++++++++++++
 2 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 07805f3f1b50b..4f5feaefc1554 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -269,8 +269,9 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                                 exchFut = exchangeFuture(exchId, evt, cache, null, msg);
                             }
                         }
-                        else
-                            exchangeFuture(msg.exchangeId(), null, null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
+                        else if (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                            exchangeFuture(msg.exchangeId(), null, null, null, null)
+                                .onAffinityChangeMessage(customEvt.eventNode(), msg);
                     }
                 }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 7e37450dd5621..771ab34116030 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -69,6 +69,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
@@ -1168,6 +1169,73 @@ public void testDelayAssignmentAffinityChanged() throws Exception {
         checkAffinity(4, topVer(4, 1), true);
     }
 
+    /**
+     * Wait for rebalance, send affinity change message, but affinity already changed (new node joined).
+     *
+     * @throws Exception If failed.
+     */
+    public void testDelayAssignmentAffinityChanged2() throws Exception {
+        Ignite ignite0 = startServer(0, 1);
+
+        TestTcpDiscoverySpi discoSpi0 =
+            (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+        TestRecordingCommunicationSpi commSpi0 =
+            (TestRecordingCommunicationSpi)ignite0.configuration().getCommunicationSpi();
+
+        startClient(1, 2);
+
+        checkAffinity(2, topVer(2, 0), true);
+
+        startServer(2, 3);
+
+        checkAffinity(3, topVer(3, 1), false);
+
+        discoSpi0.blockCustomEvent();
+
+        stopNode(2, 4);
+
+        discoSpi0.waitCustomEvent();
+
+        blockSupplySend(commSpi0, CACHE_NAME1);
+
+        final IgniteInternalFuture<?> startedFuture = multithreadedAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                startServer(3, 5);
+
+                return null;
+            }
+        }, 1, "server-starter");
+
+        Thread.sleep(2_000);
+
+        discoSpi0.stopBlock();
+
+        boolean started = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return startedFuture.isDone();
+            }
+        }, 10_000);
+
+        if (!started)
+            startedFuture.cancel();
+
+        assertTrue(started);
+
+        checkAffinity(3, topVer(5, 0), false);
+
+        checkNoExchange(3, topVer(5, 1));
+
+        commSpi0.stopBlock();
+
+        checkAffinity(3, topVer(5, 1), true);
+
+        List<GridDhtPartitionsExchangeFuture> exFutures = grid(3).context().cache().context().exchange().exchangeFutures();
+
+        for (GridDhtPartitionsExchangeFuture f : exFutures)
+            //Shouldn't contains staled futures.
+            assertTrue(f.topologyVersion().topologyVersion() >= 5);
+    }
+
     /**
      * Wait for rebalance, cache is destroyed and created again.
      *

From 6abe5bf5bd732bf9f79df577e159243520dd5c0b Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 09:47:45 +0300
Subject: [PATCH 249/516] Fixed compilation.

---
 .../internal/processors/hadoop/shuffle/HadoopShuffle.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
index 8ffea8c75075d..81001a6ff0dec 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -63,7 +63,7 @@ public class HadoopShuffle extends HadoopComponent {
         super.start(ctx);
 
         ctx.kernalContext().io().addMessageListener(GridTopic.TOPIC_HADOOP_MSG, new GridMessageListener() {
-            @Override public void onMessage(UUID nodeId, Object msg) {
+            @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
                 onMessageReceived(nodeId, (HadoopMessage)msg);
             }
         });

From 6b54f991c13efb3630527a3010c4f90105a4fc76 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 21 Jun 2017 19:09:37 +0300
Subject: [PATCH 250/516] ignite-5574 For offheap_tiered cache first try read
 value from offheap in order to avoid redundant synchronization on entry

---
 .../processors/cache/GridCacheAdapter.java    |  14 +-
 .../cache/IgniteCacheNoSyncForGetTest.java    | 163 ++++++++++++++++--
 2 files changed, 157 insertions(+), 20 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index c1b1a5cdfd4f6..f2c6410831482 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -820,7 +820,13 @@ public String toString() {
                 if (nearKey && !ctx.isNear())
                     return null;
 
-                if (modes.heap) {
+                if (modes.offheap || modes.swap) {
+                    GridCacheSwapManager swapMgr = ctx.isNear() ? ctx.near().dht().context().swap() : ctx.swap();
+
+                    cacheVal = swapMgr.readValue(cacheKey, modes.offheap, modes.swap);
+                }
+
+                if (cacheVal == null && modes.heap) {
                     GridCacheEntryEx e = nearKey ? peekEx(cacheKey) :
                         (ctx.isNear() ? ctx.near().dht().peekEx(cacheKey) : peekEx(cacheKey));
 
@@ -831,12 +837,6 @@ public String toString() {
                         modes.swap = false;
                     }
                 }
-
-                if (modes.offheap || modes.swap) {
-                    GridCacheSwapManager swapMgr = ctx.isNear() ? ctx.near().dht().context().swap() : ctx.swap();
-
-                    cacheVal = swapMgr.readValue(cacheKey, modes.offheap, modes.swap);
-                }
             }
             else
                 cacheVal = localCachePeek0(cacheKey, modes.heap, modes.offheap, modes.swap, plc);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java
index 3e624a3cf0391..f65ad11328fec 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.cache.CacheEntry;
 import org.apache.ignite.cache.CacheEntryProcessor;
 import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -108,7 +109,22 @@ public void testAtomicGetOffheap() throws Exception {
         for (boolean getAll0 : getAll) {
             for (boolean expiryPlc0 : cfgExpiryPlc)
                 for (boolean withExpiryPlc0 : withExpiryPlc)
-                    doGet(ATOMIC, OFFHEAP_TIERED, getAll0, expiryPlc0, withExpiryPlc0);
+                    doGet(ATOMIC, OFFHEAP_TIERED, false, getAll0, expiryPlc0, withExpiryPlc0);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicPeekOffheap() throws Exception {
+        boolean getAll[] = {true, false};
+        boolean cfgExpiryPlc[] = {true, false};
+        boolean withExpiryPlc[] = {true, false};
+
+        for (boolean getAll0 : getAll) {
+            for (boolean expiryPlc0 : cfgExpiryPlc)
+                for (boolean withExpiryPlc0 : withExpiryPlc)
+                    doGet(ATOMIC, OFFHEAP_TIERED, true, getAll0, expiryPlc0, withExpiryPlc0);
         }
     }
 
@@ -123,24 +139,40 @@ public void testTxGetOffheap() throws Exception {
         for (boolean getAll0 : getAll) {
             for (boolean expiryPlc0 : cfgExpiryPlc)
                 for (boolean withExpiryPlc0 : withExpiryPlc)
-                    doGet(TRANSACTIONAL, OFFHEAP_TIERED, getAll0, expiryPlc0, withExpiryPlc0);
+                    doGet(TRANSACTIONAL, OFFHEAP_TIERED, false, getAll0, expiryPlc0, withExpiryPlc0);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxPeekOffheap() throws Exception {
+        boolean getAll[] = {true, false};
+        boolean cfgExpiryPlc[] = {true, false};
+        boolean withExpiryPlc[] = {true, false};
+
+        for (boolean getAll0 : getAll) {
+            for (boolean expiryPlc0 : cfgExpiryPlc)
+                for (boolean withExpiryPlc0 : withExpiryPlc)
+                    doGet(TRANSACTIONAL, OFFHEAP_TIERED, false, getAll0, expiryPlc0, withExpiryPlc0);
         }
     }
 
     /**
      * @param atomicityMode Cache atomicity mode.
      * @param memoryMode Cache memory mode.
-     * @param getAll Test getAll flag.
+     * @param all Test all flag.
      * @param cfgExpiryPlc Configured expiry policy flag.
      * @param withExpiryPlc Custom expiry policy flag.
      * @throws Exception If failed.
      */
     private void doGet(CacheAtomicityMode atomicityMode,
         CacheMemoryMode memoryMode,
-        final boolean getAll,
+        final boolean peek,
+        final boolean all,
         final boolean cfgExpiryPlc,
         final boolean withExpiryPlc) throws Exception {
-        log.info("Test get [getAll=" + getAll + ", cfgExpiryPlc=" + cfgExpiryPlc + ']');
+        log.info("Test get [all=" + all + ", cfgExpiryPlc=" + cfgExpiryPlc + ']');
 
         Ignite srv = ignite(0);
 
@@ -163,7 +195,7 @@ private void doGet(CacheAtomicityMode atomicityMode,
 
                 IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
                     @Override public Void call() throws Exception {
-                        if (getAll)
+                        if (all)
                             cache.invokeAll(data.keySet(), new HangEntryProcessor());
                         else
                             cache.invoke(1, new HangEntryProcessor());
@@ -177,13 +209,19 @@ private void doGet(CacheAtomicityMode atomicityMode,
 
                     assertTrue(wait);
 
-                    if (getAll) {
-                        assertEquals(data, client.compute().affinityCall(cache.getName(), 1,
-                            new GetAllClosure(data.keySet(), cache.getName(), withExpiryPlc)));
+                    if (all) {
+                        IgniteCallable<Object> clo = peek ?
+                            new PeekAllClosure(data.keySet(), cache.getName(), withExpiryPlc) :
+                            new GetAllClosure(data.keySet(), cache.getName(), withExpiryPlc);
+
+                        assertEquals(data, client.compute().affinityCall(cache.getName(), 1, clo));
                     }
                     else {
-                        assertEquals(1, client.compute().affinityCall(cache.getName(), 1,
-                            new GetClosure(1, cache.getName(), withExpiryPlc)));
+                        IgniteCallable<Object> clo = peek ?
+                            new PeekClosure(1, cache.getName(), withExpiryPlc) :
+                            new GetClosure(1, cache.getName(), withExpiryPlc);
+
+                        assertEquals(1, client.compute().affinityCall(cache.getName(), 1, clo));
                     }
 
                     hangLatch.countDown();
@@ -204,7 +242,7 @@ private void doGet(CacheAtomicityMode atomicityMode,
 
                 IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
                     @Override public Void call() throws Exception {
-                        if (getAll)
+                        if (all)
                             cache.invokeAll(data.keySet(), new HangEntryProcessor());
                         else
                             cache.invoke(1, new HangEntryProcessor());
@@ -223,7 +261,7 @@ private void doGet(CacheAtomicityMode atomicityMode,
                     if (withExpiryPlc)
                         srvCache = srvCache.withExpiryPolicy(ModifiedExpiryPolicy.factoryOf(Duration.FIVE_MINUTES).create());
 
-                    if (getAll) {
+                    if (all) {
                         assertEquals(data, srvCache.getAll(data.keySet()));
                         assertEquals(data.size(), srvCache.getEntries(data.keySet()).size());
                     }
@@ -390,4 +428,103 @@ public static class GetAllClosure implements IgniteCallable<Object> {
             return vals;
         }
     }
+
+    /**
+     *
+     */
+    public static class PeekClosure implements IgniteCallable<Object> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** */
+        private final int key;
+
+        /** */
+        private final String cacheName;
+
+        /** */
+        private final boolean withExpiryPlc;
+
+        /**
+         * @param key Key.
+         * @param cacheName Cache name.
+         * @param withExpiryPlc Custom expiry policy flag.
+         */
+        PeekClosure(int key, String cacheName, boolean withExpiryPlc) {
+            this.key = key;
+            this.cacheName = cacheName;
+            this.withExpiryPlc = withExpiryPlc;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object call() throws Exception {
+            IgniteCache cache = ignite.cache(cacheName);
+
+            if (withExpiryPlc)
+                cache = cache.withExpiryPolicy(ModifiedExpiryPolicy.factoryOf(Duration.FIVE_MINUTES).create());
+
+            Object val = cache.localPeek(key, CachePeekMode.ALL);
+
+            CacheEntry e = cache.getEntry(key);
+
+            assertEquals(val, e.getValue());
+
+            return val;
+        }
+    }
+
+    /**
+     *
+     */
+    public static class PeekAllClosure implements IgniteCallable<Object> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** */
+        private final Set<Object> keys;
+
+        /** */
+        private final String cacheName;
+
+        /** */
+        private final boolean withExpiryPlc;
+
+        /**
+         * @param keys Keys.
+         * @param cacheName Cache name.
+         * @param withExpiryPlc Custom expiry policy flag.
+         */
+        PeekAllClosure(Set<Object> keys, String cacheName, boolean withExpiryPlc) {
+            this.keys = keys;
+            this.cacheName = cacheName;
+            this.withExpiryPlc = withExpiryPlc;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object call() throws Exception {
+            IgniteCache cache = ignite.cache(cacheName);
+
+            if (withExpiryPlc)
+                cache = cache.withExpiryPolicy(ModifiedExpiryPolicy.factoryOf(Duration.FIVE_MINUTES).create());
+
+            Map vals = new HashMap();
+
+            for (Object key : keys)
+                vals.put(key, cache.localPeek(key, CachePeekMode.ALL));
+
+            Collection<CacheEntry> entries = cache.getEntries(keys);
+
+            assertEquals(vals.size(), entries.size());
+
+            for (CacheEntry entry : entries) {
+                Object val = vals.get(entry.getKey());
+
+                assertEquals(val, entry.getValue());
+            }
+
+            return vals;
+        }
+    }
 }

From 7d5217260b293b7224340349b5e44792999600f3 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 11:47:21 +0300
Subject: [PATCH 251/516] Rethrow handshake exceptions as it done for
 ConnectionException.

---
 .../tcp/TcpCommunicationSpi.java              | 32 +++++++++++++------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 10d2141ffdf15..0f910df798947 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3009,7 +3009,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
 
                     // Reconnect for the second time, if connection is not established.
                     if (!failureDetThrReached && connectAttempts < 5 &&
-                        (X.hasCause(e, ConnectException.class) || X.hasCause(e, SocketTimeoutException.class))) {
+                        (X.hasCause(e, ConnectException.class, HandshakeException.class, SocketTimeoutException.class))) {
                         U.sleep(200);
 
                         connectAttempts++;
@@ -3036,7 +3036,8 @@ else if (X.hasCause(e, SocketTimeoutException.class))
 
             if (enableForcibleNodeKill) {
                 if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
-                    X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
+                    X.hasCause(errs, ConnectException.class, HandshakeException.class,
+                        SocketTimeoutException.class, HandshakeTimeoutException.class,
                         IgniteSpiOperationTimeoutException.class)) {
                     U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
                         "cluster [" + "rmtNode=" + node + ']', errs);
@@ -3048,7 +3049,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                 }
             }
 
-            if (X.hasCause(errs, ConnectException.class))
+            if (X.hasCause(errs, ConnectException.class, HandshakeException.class))
                 throw errs;
         }
 
@@ -3101,7 +3102,7 @@ private <T> long safeHandshake(
                         sslHnd = new BlockingSslHandler(sslMeta.sslEngine(), ch, directBuf, ByteOrder.nativeOrder(), log);
 
                         if (!sslHnd.handshake())
-                            throw new IgniteCheckedException("SSL handshake is not completed.");
+                            throw new HandshakeException("SSL handshake is not completed.");
 
                         ByteBuffer handBuff = sslHnd.applicationBuffer();
 
@@ -3111,7 +3112,7 @@ private <T> long safeHandshake(
                             int read = ch.read(buf);
 
                             if (read == -1)
-                                throw new IgniteCheckedException("Failed to read remote node ID (connection closed).");
+                                throw new HandshakeException("Failed to read remote node ID (connection closed).");
 
                             buf.flip();
 
@@ -3127,7 +3128,7 @@ private <T> long safeHandshake(
                             int read = ch.read(buf);
 
                             if (read == -1)
-                                throw new IgniteCheckedException("Failed to read remote node ID (connection closed).");
+                                throw new HandshakeException("Failed to read remote node ID (connection closed).");
 
                             i += read;
                         }
@@ -3136,7 +3137,7 @@ private <T> long safeHandshake(
                     UUID rmtNodeId0 = U.bytesToUuid(buf.array(), 1);
 
                     if (!rmtNodeId.equals(rmtNodeId0))
-                        throw new IgniteCheckedException("Remote node ID is not as expected [expected=" + rmtNodeId +
+                        throw new HandshakeException("Remote node ID is not as expected [expected=" + rmtNodeId +
                             ", rcvd=" + rmtNodeId0 + ']');
                     else if (log.isDebugEnabled())
                         log.debug("Received remote node ID: " + rmtNodeId0);
@@ -3222,7 +3223,7 @@ else if (log.isDebugEnabled())
                                 int read = ch.read(buf);
 
                                 if (read == -1)
-                                    throw new IgniteCheckedException("Failed to read remote node recovery handshake " +
+                                    throw new HandshakeException("Failed to read remote node recovery handshake " +
                                         "(connection closed).");
 
                                 buf.flip();
@@ -3260,7 +3261,7 @@ else if (log.isDebugEnabled())
                                 int read = ch.read(buf);
 
                                 if (read == -1)
-                                    throw new IgniteCheckedException("Failed to read remote node recovery handshake " +
+                                    throw new HandshakeException("Failed to read remote node recovery handshake " +
                                         "(connection closed).");
 
                                 i += read;
@@ -3480,6 +3481,19 @@ private NodeIdMessage nodeIdMessage() {
         return S.toString(TcpCommunicationSpi.class, this);
     }
 
+    /** Internal exception class for proper timeout handling. */
+    private static class HandshakeException extends IgniteCheckedException {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param msg Error message.
+         */
+        HandshakeException(String msg) {
+            super(msg);
+        }
+    }
+
     /** Internal exception class for proper timeout handling. */
     private static class HandshakeTimeoutException extends IgniteCheckedException {
         /** */

From 81cdea40743131cac9dae49150c1038073595f7e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 13:00:30 +0300
Subject: [PATCH 252/516] Fixed tests.

---
 .../IgniteClientReconnectAbstractTest.java        | 10 +++++-----
 ...iteClientReconnectContinuousProcessorTest.java | 15 +--------------
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
index a793760f44fbf..17a1ad41ae448 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
@@ -69,9 +70,6 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
     /** */
     private static final long RECONNECT_TIMEOUT = 10_000;
 
-    /** Reconnect should occurs before failure detection time is out. */
-    public static final long FAILURE_DETECTION_TIMEOUT = RECONNECT_TIMEOUT +2000;
-
     /** */
     protected boolean clientMode;
 
@@ -79,8 +77,6 @@ public abstract class IgniteClientReconnectAbstractTest extends GridCommonAbstra
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
-        cfg.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT);
-
         TestTcpDiscoverySpi disco = new TestTcpDiscoverySpi();
 
         disco.setIpFinder(ipFinder);
@@ -157,6 +153,8 @@ protected BlockTcpCommunicationSpi commSpi(Ignite ignite) {
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL, "true");
+
         int srvs = serverCount();
 
         if (srvs > 0)
@@ -177,6 +175,8 @@ protected BlockTcpCommunicationSpi commSpi(Ignite ignite) {
     @Override protected void afterTestsStopped() throws Exception {
         super.afterTestsStopped();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         stopAllGrids();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
index 0ff5883d11b57..c86dcabb3a208 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
@@ -23,6 +23,7 @@
 import javax.cache.event.CacheEntryUpdatedListener;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.query.ContinuousQuery;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -110,20 +111,6 @@ public void testEventListenerReconnect() throws Exception {
         assertFalse(lsnr.latch.await(3000, MILLISECONDS));
     }
 
-    /**
-     * @throws Exception If failed.
-     */
-    public void testMessageListenerReconnectAndStopFromServer() throws Exception {
-        testMessageListenerReconnect(false);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testMessageListenerReconnectAndStopFromClient() throws Exception {
-        testMessageListenerReconnect(true);
-    }
-
     /**
      * @param stopFromClient If {@code true} stops listener from client node, otherwise from server.
      * @throws Exception If failed.

From 473abcafca568c7c898b0b1ae91fe964084fdf43 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 28 Jun 2017 14:49:49 +0300
Subject: [PATCH 253/516] Tests fixed

---
 .../IgniteCachePutRetryAbstractSelfTest.java  | 51 +++----------------
 1 file changed, 6 insertions(+), 45 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
index fc9017927d3bc..0c2b868704cd3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
@@ -57,7 +57,6 @@
 import org.apache.ignite.spi.swapspace.inmemory.GridTestSwapSpaceSpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -631,71 +630,33 @@ public SetEntryProcessor(Integer val) {
         }
     }
 
-    /**
-     *
-     */
     private static class TestStoreFactory implements Factory<CacheStore> {
         /** {@inheritDoc} */
-        @Override public CacheStore create() {
-            return new CacheStoreAdapter() {
-                @Override public Object load(Object key) throws CacheLoaderException {
-                    return null;
-                }
-
-                @Override public void write(Cache.Entry entry) throws CacheWriterException {
-                    // No-op.
-                }
-
-                @Override public void delete(Object key) throws CacheWriterException {
-                    // No-op.
-                }
-            };
-        }
-    }
-
-
-/*
-    private static class TestStoreFactory implements Factory<CacheStore> {
-        */
-/** {@inheritDoc} *//*
-
         @Override public CacheStore create() {
             return new TestCacheStore();
         }
     }
 
-    */
-/**
+    /**
      *
-     *//*
-
+     */
     private static class TestCacheStore extends CacheStoreAdapter {
-        */
-/** Store map. *//*
-
+        /** Store map. */
         private static Map STORE_MAP = new ConcurrentHashMap();
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public Object load(Object key) throws CacheLoaderException {
             return STORE_MAP.get(key);
         }
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public void write(Cache.Entry entry) throws CacheWriterException {
             STORE_MAP.put(entry.getKey(), entry.getValue());
         }
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public void delete(Object key) throws CacheWriterException {
             STORE_MAP.remove(key);
         }
     }
-*/
-
 }
\ No newline at end of file

From 4f383ae772631987c3f4ac29bb654b4811fbc407 Mon Sep 17 00:00:00 2001
From: agura <agura@apache.org>
Date: Wed, 28 Jun 2017 14:49:49 +0300
Subject: [PATCH 254/516] Tests fixed.

---
 .../IgniteCachePutRetryAbstractSelfTest.java  | 51 +++----------------
 1 file changed, 6 insertions(+), 45 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
index fc9017927d3bc..0c2b868704cd3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePutRetryAbstractSelfTest.java
@@ -57,7 +57,6 @@
 import org.apache.ignite.spi.swapspace.inmemory.GridTestSwapSpaceSpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -631,71 +630,33 @@ public SetEntryProcessor(Integer val) {
         }
     }
 
-    /**
-     *
-     */
     private static class TestStoreFactory implements Factory<CacheStore> {
         /** {@inheritDoc} */
-        @Override public CacheStore create() {
-            return new CacheStoreAdapter() {
-                @Override public Object load(Object key) throws CacheLoaderException {
-                    return null;
-                }
-
-                @Override public void write(Cache.Entry entry) throws CacheWriterException {
-                    // No-op.
-                }
-
-                @Override public void delete(Object key) throws CacheWriterException {
-                    // No-op.
-                }
-            };
-        }
-    }
-
-
-/*
-    private static class TestStoreFactory implements Factory<CacheStore> {
-        */
-/** {@inheritDoc} *//*
-
         @Override public CacheStore create() {
             return new TestCacheStore();
         }
     }
 
-    */
-/**
+    /**
      *
-     *//*
-
+     */
     private static class TestCacheStore extends CacheStoreAdapter {
-        */
-/** Store map. *//*
-
+        /** Store map. */
         private static Map STORE_MAP = new ConcurrentHashMap();
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public Object load(Object key) throws CacheLoaderException {
             return STORE_MAP.get(key);
         }
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public void write(Cache.Entry entry) throws CacheWriterException {
             STORE_MAP.put(entry.getKey(), entry.getValue());
         }
 
-        */
-/** {@inheritDoc} *//*
-
+        /** {@inheritDoc} */
         @Override public void delete(Object key) throws CacheWriterException {
             STORE_MAP.remove(key);
         }
     }
-*/
-
 }
\ No newline at end of file

From 4b11bb27d5caeb7ba7bc2929685d7c548f4c327d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 15:45:42 +0300
Subject: [PATCH 255/516] Tests fixed.

---
 .../cache/IgniteDynamicCacheStartSelfTest.java    | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
index 48e06ee7c82a0..49a49c7958e94 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheExistsException;
 import org.apache.ignite.cache.CacheMode;
@@ -144,6 +145,20 @@ public int nodeCount() {
         stopAllGrids();
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        if(grid(0).cache(DYNAMIC_CACHE_NAME)!=null)
+            grid(0).destroyCache(DYNAMIC_CACHE_NAME);
+
+        while(Ignition.allGrids().size() > nodeCount()) {
+            stopGrid(nodeCount());
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From 9b92dac5756619ed5218db269acdae1ef02ef8ae Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 18:20:02 +0300
Subject: [PATCH 256/516] Tests fixed.

---
 .../ignite/spi/communication/tcp/TcpCommunicationSpi.java       | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 0f910df798947..122700e84785c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -23,6 +23,7 @@
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -2917,6 +2918,7 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
                     }
 
                     if (failureDetectionTimeoutEnabled() && (e instanceof HandshakeTimeoutException ||
+                        X.hasCause(e, SocketException.class) ||
                         timeoutHelper.checkFailureTimeoutReached(e))) {
 
                         String msg = "Handshake timed out (failure detection timeout is reached) " +

From 0295a1a7ecb2ef57a917ddc9015ff8b71a6ddb14 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Fri, 23 Jun 2017 21:00:45 +0300
Subject: [PATCH 257/516] IGNITE-5521: Large near caches lead to cluster
 instability with metrics enabled.

---
 .../near/GridNearCacheAdapter.java            |  2 +-
 .../near/GridCacheNearEvictionSelfTest.java   |  5 ++--
 .../near/GridCacheNearMetricsSelfTest.java    | 30 ++++++++++++++++++-
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
index 4ddad74157987..8dfe1698991a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
@@ -308,7 +308,7 @@ protected void processGetResponse(UUID nodeId, GridNearGetResponse res) {
 
     /** {@inheritDoc} */
     @Override public int size() {
-        return nearEntries().size() + dht().size();
+        return dht().size();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
index df5943611075c..6181f50962003 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
@@ -165,12 +165,13 @@ public void testNearEnabledThreeNodes() throws Exception {
 
             for (int i = 0; i < gridCnt; i++) {
                 final GridCacheAdapter cache = internalCache(i);
+                final GridCacheAdapter near =  near(i);
 
                 // Repeatedly check cache sizes because of concurrent cache updates.
                 assertTrue(GridTestUtils.waitForCondition(new PA() {
                     @Override public boolean apply() {
                         // Every node contains either near, backup, or primary.
-                        return cnt == cache.size();
+                        return cnt == cache.size() + near.nearSize();
                     }
                 }, getTestTimeout()));
 
@@ -183,4 +184,4 @@ public void testNearEnabledThreeNodes() throws Exception {
             stopAllGrids();
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
index 09ff519a67976..de2696f56392a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
@@ -21,7 +21,10 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -97,6 +100,31 @@ protected int keyCount() {
         return cc;
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNearCacheDoesNotAffectCacheSize() throws Exception {
+        IgniteCache<Integer, Integer> cache0 = grid(0).cache(null);
+
+        for (int i = 0; i < 100 ; i++)
+            cache0.put(i, i);
+
+        IgniteEx g1 = grid(1);
+
+        IgniteCache<Integer, Integer> cache1 = g1.cache(null    );
+
+        ClusterNode localNode = g1.cluster().localNode();
+
+        int beforeSize = cache1.localMetrics().getSize();
+
+        for (int i = 0; i < 100 ; i++) {
+            if (!affinity(cache1).isPrimaryOrBackup(localNode, i))
+                cache1.get(i); // put entry to near cache
+        }
+
+        assertEquals(beforeSize, cache1.localMetrics().getSize());
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -417,4 +445,4 @@ else if (affinity(jcache).isBackup(g.cluster().localNode(), key)){
             }
         }
     }
-}
\ No newline at end of file
+}

From 16fed552fc8f91de550207eeebd5850e685960a8 Mon Sep 17 00:00:00 2001
From: AMRepo <andrey.mashenkov@gmail.com>
Date: Thu, 29 Jun 2017 10:49:34 +0300
Subject: [PATCH 258/516] Fix tests.

---
 ...PartitionedBackupNodeFailureRecoveryTest.java | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java
index 6654fd93a77a8..e1ef59c5b79be 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java
@@ -33,6 +33,8 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest;
+import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.testframework.GridTestUtils;
 
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.PRIMARY;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -148,16 +150,22 @@ public void testBackUpFail() throws Exception {
 
                         IgniteEx backUp = startGrid(2);
 
-                        IgniteCache<Integer, Integer> cache3 = backUp.cache(null);
+                        final IgniteCache<Integer, Integer> cache3 = backUp.cache(null);
 
                         lock.lock();
 
                         try {
-                            Integer backUpVal = cache3.localPeek(finalKey);
+                            boolean res = GridTestUtils.waitForCondition(new PA() {
+                                @Override public boolean apply() {
+                                    Integer actl = cache3.localPeek(finalKey);
 
-                            Integer exp = cntr.get();
+                                    Integer exp = cntr.get();
 
-                            assertEquals(exp, backUpVal);
+                                    return exp.equals(actl);
+                                }
+                            }, 1000);
+
+                            assertTrue(res);
                         }
                         finally {
                             lock.unlock();

From 65d93e48c264f4bfff0a94856fdfeb83375a3976 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Fri, 23 Jun 2017 21:00:45 +0300
Subject: [PATCH 259/516] IGNITE-5521: Large near caches lead to cluster
 instability with metrics enabled

---
 .../near/GridNearCacheAdapter.java            |  2 +-
 .../near/GridCacheNearEvictionSelfTest.java   |  5 ++--
 .../near/GridCacheNearMetricsSelfTest.java    | 30 ++++++++++++++++++-
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
index 4ddad74157987..8dfe1698991a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
@@ -308,7 +308,7 @@ protected void processGetResponse(UUID nodeId, GridNearGetResponse res) {
 
     /** {@inheritDoc} */
     @Override public int size() {
-        return nearEntries().size() + dht().size();
+        return dht().size();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
index df5943611075c..6181f50962003 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionSelfTest.java
@@ -165,12 +165,13 @@ public void testNearEnabledThreeNodes() throws Exception {
 
             for (int i = 0; i < gridCnt; i++) {
                 final GridCacheAdapter cache = internalCache(i);
+                final GridCacheAdapter near =  near(i);
 
                 // Repeatedly check cache sizes because of concurrent cache updates.
                 assertTrue(GridTestUtils.waitForCondition(new PA() {
                     @Override public boolean apply() {
                         // Every node contains either near, backup, or primary.
-                        return cnt == cache.size();
+                        return cnt == cache.size() + near.nearSize();
                     }
                 }, getTestTimeout()));
 
@@ -183,4 +184,4 @@ public void testNearEnabledThreeNodes() throws Exception {
             stopAllGrids();
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
index 09ff519a67976..de2696f56392a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
@@ -21,7 +21,10 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -97,6 +100,31 @@ protected int keyCount() {
         return cc;
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNearCacheDoesNotAffectCacheSize() throws Exception {
+        IgniteCache<Integer, Integer> cache0 = grid(0).cache(null);
+
+        for (int i = 0; i < 100 ; i++)
+            cache0.put(i, i);
+
+        IgniteEx g1 = grid(1);
+
+        IgniteCache<Integer, Integer> cache1 = g1.cache(null    );
+
+        ClusterNode localNode = g1.cluster().localNode();
+
+        int beforeSize = cache1.localMetrics().getSize();
+
+        for (int i = 0; i < 100 ; i++) {
+            if (!affinity(cache1).isPrimaryOrBackup(localNode, i))
+                cache1.get(i); // put entry to near cache
+        }
+
+        assertEquals(beforeSize, cache1.localMetrics().getSize());
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -417,4 +445,4 @@ else if (affinity(jcache).isBackup(g.cluster().localNode(), key)){
             }
         }
     }
-}
\ No newline at end of file
+}

From ff1afad1278e939aef71b274e959fd4256904971 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 27 Jun 2017 10:34:42 +0300
Subject: [PATCH 260/516] Partially reverted GG-12352.

(cherry picked from commit 3668b91)
---
 .../ignite/spi/discovery/tcp/ServerImpl.java      | 10 +---------
 ...IgniteBinaryMetadataUpdateNodeRestartTest.java | 10 ++++++++++
 .../IgniteCacheNearRestartRollbackSelfTest.java   | 15 +++++++++++++++
 ...riteSynchronizationModesMultithreadedTest.java |  5 +++++
 4 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 2f7e9b470aae6..6e5af2986dcff 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1242,7 +1242,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
                 errs.add(e);
 
-                if (X.hasCause(e, SSLException.class)) {
+                if (X.hasCause(e, SSLException.class, StreamCorruptedException.class)) {
                     if (--sslConnectAttempts == 0)
                         throw new IgniteException("Unable to establish secure connection. " +
                             "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
@@ -1250,14 +1250,6 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                     continue;
                 }
 
-                if (X.hasCause(e, StreamCorruptedException.class)) {
-                    if (--sslConnectAttempts == 0)
-                        throw new IgniteException("Unable to establish plain connection. " +
-                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
-
-                    continue;
-                }
-
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
index 814fb08eacba5..3d552847fd71d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteBinaryMetadataUpdateNodeRestartTest.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheEntryProcessor;
 import org.apache.ignite.cluster.ClusterTopologyException;
@@ -88,10 +89,19 @@ public class IgniteBinaryMetadataUpdateNodeRestartTest extends GridCommonAbstrac
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         super.afterTestsStopped();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index a48497d8f39c4..3f242b5726867 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -31,6 +31,7 @@
 import javax.cache.processor.MutableEntry;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -92,6 +93,20 @@ public class IgniteCacheNearRestartRollbackSelfTest extends GridCommonAbstractTe
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+    }
+
     /**
      * @param gridName Grid name.
      * @return Cache configuration.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
index 08396daa5efeb..c214a772d422c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cache.store.CacheStoreAdapter;
@@ -104,6 +105,8 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true");
+
         startGrids(SRVS);
 
         clientMode = true;
@@ -119,6 +122,8 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         super.afterTestsStopped();
     }
 

From 7fbb95d1698c534957c9f0b7c5558f43985cfe68 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 27 Jun 2017 20:43:07 +0300
Subject: [PATCH 261/516] GG-12370: Fixed massive NODE_FAILED events lead to
 excessive momentary memory consumption.

(cherry picked from commit 0521b77)
---
 .../GridCachePartitionExchangeManager.java    |  5 +-
 .../CacheLateAffinityAssignmentTest.java      | 68 +++++++++++++++++++
 2 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 92142c0731240..3df0bafc06c25 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -269,8 +269,9 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                                 exchFut = exchangeFuture(exchId, e, null, msg);
                             }
                         }
-                        else
-                            exchangeFuture(msg.exchangeId(), null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
+                        else if (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                            exchangeFuture(msg.exchangeId(), null, null, null)
+                                .onAffinityChangeMessage(customEvt.eventNode(), msg);
                     }
                 }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 7e37450dd5621..771ab34116030 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -69,6 +69,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
@@ -1168,6 +1169,73 @@ public void testDelayAssignmentAffinityChanged() throws Exception {
         checkAffinity(4, topVer(4, 1), true);
     }
 
+    /**
+     * Wait for rebalance, send affinity change message, but affinity already changed (new node joined).
+     *
+     * @throws Exception If failed.
+     */
+    public void testDelayAssignmentAffinityChanged2() throws Exception {
+        Ignite ignite0 = startServer(0, 1);
+
+        TestTcpDiscoverySpi discoSpi0 =
+            (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+        TestRecordingCommunicationSpi commSpi0 =
+            (TestRecordingCommunicationSpi)ignite0.configuration().getCommunicationSpi();
+
+        startClient(1, 2);
+
+        checkAffinity(2, topVer(2, 0), true);
+
+        startServer(2, 3);
+
+        checkAffinity(3, topVer(3, 1), false);
+
+        discoSpi0.blockCustomEvent();
+
+        stopNode(2, 4);
+
+        discoSpi0.waitCustomEvent();
+
+        blockSupplySend(commSpi0, CACHE_NAME1);
+
+        final IgniteInternalFuture<?> startedFuture = multithreadedAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                startServer(3, 5);
+
+                return null;
+            }
+        }, 1, "server-starter");
+
+        Thread.sleep(2_000);
+
+        discoSpi0.stopBlock();
+
+        boolean started = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return startedFuture.isDone();
+            }
+        }, 10_000);
+
+        if (!started)
+            startedFuture.cancel();
+
+        assertTrue(started);
+
+        checkAffinity(3, topVer(5, 0), false);
+
+        checkNoExchange(3, topVer(5, 1));
+
+        commSpi0.stopBlock();
+
+        checkAffinity(3, topVer(5, 1), true);
+
+        List<GridDhtPartitionsExchangeFuture> exFutures = grid(3).context().cache().context().exchange().exchangeFutures();
+
+        for (GridDhtPartitionsExchangeFuture f : exFutures)
+            //Shouldn't contains staled futures.
+            assertTrue(f.topologyVersion().topologyVersion() >= 5);
+    }
+
     /**
      * Wait for rebalance, cache is destroyed and created again.
      *

From 710db327c027d43c0984b7007447cecca71f3cca Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 11:47:21 +0300
Subject: [PATCH 262/516] Rethrow handshake exceptions as it done for
 ConnectionException.

(cherry picked from commit 7d52172)
---
 .../tcp/TcpCommunicationSpi.java              | 32 +++++++++++++------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 1bfa56accf2e0..9925b3d0c8736 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2572,7 +2572,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
 
                     // Reconnect for the second time, if connection is not established.
                     if (!failureDetThrReached && connectAttempts < 5 &&
-                        (X.hasCause(e, ConnectException.class) || X.hasCause(e, SocketTimeoutException.class))) {
+                        (X.hasCause(e, ConnectException.class, HandshakeException.class, SocketTimeoutException.class))) {
                         U.sleep(200);
 
                         connectAttempts++;
@@ -2599,7 +2599,8 @@ else if (X.hasCause(e, SocketTimeoutException.class))
 
             if (enableForcibleNodeKill) {
                 if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
-                    X.hasCause(errs, ConnectException.class, SocketTimeoutException.class, HandshakeTimeoutException.class,
+                    X.hasCause(errs, ConnectException.class, HandshakeException.class,
+                        SocketTimeoutException.class, HandshakeTimeoutException.class,
                         IgniteSpiOperationTimeoutException.class)) {
                     U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
                         "cluster [" + "rmtNode=" + node + ']', errs);
@@ -2611,7 +2612,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                 }
             }
 
-            if (X.hasCause(errs, ConnectException.class))
+            if (X.hasCause(errs, ConnectException.class, HandshakeException.class))
                 throw errs;
         }
 
@@ -2662,7 +2663,7 @@ private <T> long safeHandshake(
                         sslHnd = new BlockingSslHandler(sslMeta.sslEngine(), ch, directBuf, ByteOrder.nativeOrder(), log);
 
                         if (!sslHnd.handshake())
-                            throw new IgniteCheckedException("SSL handshake is not completed.");
+                            throw new HandshakeException("SSL handshake is not completed.");
 
                         ByteBuffer handBuff = sslHnd.applicationBuffer();
 
@@ -2672,7 +2673,7 @@ private <T> long safeHandshake(
                             int read = ch.read(buf);
 
                             if (read == -1)
-                                throw new IgniteCheckedException("Failed to read remote node ID (connection closed).");
+                                throw new HandshakeException("Failed to read remote node ID (connection closed).");
 
                             buf.flip();
 
@@ -2688,7 +2689,7 @@ private <T> long safeHandshake(
                             int read = ch.read(buf);
 
                             if (read == -1)
-                                throw new IgniteCheckedException("Failed to read remote node ID (connection closed).");
+                                throw new HandshakeException("Failed to read remote node ID (connection closed).");
 
                             i += read;
                         }
@@ -2697,7 +2698,7 @@ private <T> long safeHandshake(
                     UUID rmtNodeId0 = U.bytesToUuid(buf.array(), 1);
 
                     if (!rmtNodeId.equals(rmtNodeId0))
-                        throw new IgniteCheckedException("Remote node ID is not as expected [expected=" + rmtNodeId +
+                        throw new HandshakeException("Remote node ID is not as expected [expected=" + rmtNodeId +
                             ", rcvd=" + rmtNodeId0 + ']');
                     else if (log.isDebugEnabled())
                         log.debug("Received remote node ID: " + rmtNodeId0);
@@ -2768,7 +2769,7 @@ else if (log.isDebugEnabled())
                                 int read = ch.read(buf);
 
                                 if (read == -1)
-                                    throw new IgniteCheckedException("Failed to read remote node recovery handshake " +
+                                    throw new HandshakeException("Failed to read remote node recovery handshake " +
                                         "(connection closed).");
 
                                 buf.flip();
@@ -2802,7 +2803,7 @@ else if (log.isDebugEnabled())
                                 int read = ch.read(buf);
 
                                 if (read == -1)
-                                    throw new IgniteCheckedException("Failed to read remote node recovery handshake " +
+                                    throw new HandshakeException("Failed to read remote node recovery handshake " +
                                         "(connection closed).");
 
                                 i += read;
@@ -2990,6 +2991,19 @@ private ClientKey(UUID nodeId, long order) {
         }
     }
 
+    /** Internal exception class for proper timeout handling. */
+    private static class HandshakeException extends IgniteCheckedException {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param msg Error message.
+         */
+        HandshakeException(String msg) {
+            super(msg);
+        }
+    }
+
     /** Internal exception class for proper timeout handling. */
     private static class HandshakeTimeoutException extends IgniteCheckedException {
         /** */

From 7c619f77009cc6876405839f9f6201d332d94b47 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 28 Jun 2017 15:45:42 +0300
Subject: [PATCH 263/516] Fixed tests.

(cherry picked from commit 81cdea4)
---
 .../IgniteClientReconnectAbstractTest.java        |  5 +++++
 ...iteClientReconnectContinuousProcessorTest.java | 15 +--------------
 .../cache/IgniteDynamicCacheStartSelfTest.java    | 15 +++++++++++++++
 3 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
index 4d49366e1ad23..17a1ad41ae448 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
@@ -152,6 +153,8 @@ protected BlockTcpCommunicationSpi commSpi(Ignite ignite) {
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL, "true");
+
         int srvs = serverCount();
 
         if (srvs > 0)
@@ -172,6 +175,8 @@ protected BlockTcpCommunicationSpi commSpi(Ignite ignite) {
     @Override protected void afterTestsStopped() throws Exception {
         super.afterTestsStopped();
 
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
+
         stopAllGrids();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
index 0ff5883d11b57..c86dcabb3a208 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
@@ -23,6 +23,7 @@
 import javax.cache.event.CacheEntryUpdatedListener;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.query.ContinuousQuery;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -110,20 +111,6 @@ public void testEventListenerReconnect() throws Exception {
         assertFalse(lsnr.latch.await(3000, MILLISECONDS));
     }
 
-    /**
-     * @throws Exception If failed.
-     */
-    public void testMessageListenerReconnectAndStopFromServer() throws Exception {
-        testMessageListenerReconnect(false);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testMessageListenerReconnectAndStopFromClient() throws Exception {
-        testMessageListenerReconnect(true);
-    }
-
     /**
      * @param stopFromClient If {@code true} stops listener from client node, otherwise from server.
      * @throws Exception If failed.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
index 48e06ee7c82a0..49a49c7958e94 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheExistsException;
 import org.apache.ignite.cache.CacheMode;
@@ -144,6 +145,20 @@ public int nodeCount() {
         stopAllGrids();
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        if(grid(0).cache(DYNAMIC_CACHE_NAME)!=null)
+            grid(0).destroyCache(DYNAMIC_CACHE_NAME);
+
+        while(Ignition.allGrids().size() > nodeCount()) {
+            stopGrid(nodeCount());
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From 5219a35208b95a2d7dea3825dd4cb4edc74b3542 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Mon, 26 Jun 2017 15:37:26 +0300
Subject: [PATCH 264/516] IGNITE-5076: improved multi-threaded start of nodes

---
 .../ignite/testframework/junits/GridAbstractTest.java  | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
 mode change 100644 => 100755 modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
old mode 100644
new mode 100755
index ffaec4fbc89af..5c43ecbfc9bb2
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -646,12 +646,14 @@ protected final Ignite startGrids(int cnt) throws Exception {
      * @throws Exception If failed.
      */
     protected Ignite startGridsMultiThreaded(int cnt) throws Exception {
-        if (cnt == 1)
-            return startGrids(1);
+        assert cnt > 0 : "Number of grids must be a positive number";
 
-        Ignite ignite = startGridsMultiThreaded(0, cnt);
+        Ignite ignite = startGrids(1);
 
-        checkTopology(cnt);
+        if (cnt > 1) {
+            startGridsMultiThreaded(1, cnt - 1);
+            checkTopology(cnt);
+        }
 
         return ignite;
     }

From a4fc555b118d91ec0348154b88764f010dbbae52 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 30 Jun 2017 14:38:54 +0300
Subject: [PATCH 265/516] Fixed "IGNITE-5424 GridServiceProxy does not unwraps
 exception message from InvocationTargetException." This closes #2168

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../processors/service/GridServiceProxy.java  |  9 ++-
 .../GridServiceProcessorProxySelfTest.java    | 66 +++++++++++++++++++
 2 files changed, 74 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index 2286cff9717aa..6e8fb4e0ae667 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -23,6 +23,7 @@
 import java.io.ObjectOutput;
 import java.io.Serializable;
 import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.util.ArrayList;
@@ -403,7 +404,13 @@ private ServiceProxyCallable(String mtdName, String svcName, Class[] argTypes, O
             if (mtd == null)
                 throw new GridServiceMethodNotFoundException(svcName, mtdName, argTypes);
 
-            return mtd.invoke(svcCtx.service(), args);
+            try {
+                return mtd.invoke(svcCtx.service(), args);
+            }
+            catch (InvocationTargetException e) {
+                // Get error message.
+                throw new IgniteCheckedException(e.getCause().getMessage(), e);
+            }
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
index 6fc7e02e42946..850cb5702b5b0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
@@ -18,12 +18,15 @@
 package org.apache.ignite.internal.processors.service;
 
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
 
 /**
  * Service proxy test.
@@ -63,6 +66,31 @@ public void testNodeSingletonProxy() throws Exception {
         }
     }
 
+    /**
+     * Unwraps error message from InvocationTargetException.
+     *
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    public void testException() throws Exception {
+        String name = "errorService";
+
+        Ignite ignite = grid(0);
+
+        ignite.services(ignite.cluster().forRemotes()).deployNodeSingleton(name, new ErrorServiceImpl());
+
+        final ErrorService svc = ignite.services().serviceProxy(name, ErrorService.class, false);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                svc.go();
+
+                return null;
+            }
+        }, IgniteException.class, "Test exception");
+
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -353,6 +381,7 @@ protected static class MapServiceImpl<K, V> implements MapService<K, V>, Service
             map.clear();
         }
 
+        /** {@inheritDoc} */
         @Override public int size() {
             return map.size();
         }
@@ -372,4 +401,41 @@ protected static class MapServiceImpl<K, V> implements MapService<K, V>, Service
             X.println("Executing cache service: " + ctx.name());
         }
     }
+
+    /**
+     *
+     */
+    protected interface ErrorService extends Service {
+        /**
+         *
+         */
+        void go() throws Exception;
+    }
+
+    /**
+     *
+     */
+    protected class ErrorServiceImpl implements ErrorService {
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(ServiceContext ctx) throws Exception {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void go() throws Exception {
+            throw new Exception("Test exception");
+        }
+    }
+
+
 }
\ No newline at end of file

From 62876fefc8dbd334db9dc6741c2bc91b3409343d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 30 Jun 2017 14:45:18 +0300
Subject: [PATCH 266/516] IGNITE-5473 partial fix: Create ignite
 troubleshooting logger.

---
 .../apache/ignite/IgniteSystemProperties.java |  7 +++++++
 .../tcp/TcpCommunicationSpi.java              | 20 ++++++++++++++-----
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 24df237db6a75..aaba91e15d304 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -136,6 +136,13 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_QUIET = "IGNITE_QUIET";
 
+    /**
+     * Setting this option to {@code true} will enable troubleshooting logger.
+     * Troubleshooting logger makes logging more verbose without enabling debug mode
+     * to provide more detailed logs without performance penalty.
+     */
+    public static final String IGNITE_TROUBLESHOOTING_LOGGER = "IGNITE_TROUBLESHOOTING_LOGGER";
+
     /**
      * Setting to {@code true} enables writing sensitive information in {@code toString()} output.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 122700e84785c..7c49a35b5c62a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -349,6 +349,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     private boolean enableForcibleNodeKill = IgniteSystemProperties
         .getBoolean(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
 
+    /** */
+    private boolean enableTroubleshootingLog = IgniteSystemProperties
+        .getBoolean(IgniteSystemProperties.IGNITE_TROUBLESHOOTING_LOGGER);
+
     /** Server listener. */
     private final GridNioServerListener<Message> srvLsnr =
         new GridNioServerListenerAdapter<Message>() {
@@ -2989,9 +2993,10 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
                     boolean failureDetThrReached = timeoutHelper.checkFailureTimeoutReached(e);
 
-                    U.error(log, "Failed to establish connection to a remote node [node=" + node +
-                        ", addr=" + addr + ", connectAttempts=" + connectAttempts +
-                        ", failureDetThrReached" + failureDetThrReached + ']', e);
+                    if (enableTroubleshootingLog)
+                        U.error(log, "Failed to establish connection to a remote node [node=" + node +
+                            ", addr=" + addr + ", connectAttempts=" + connectAttempts +
+                            ", failureDetThrReached=" + failureDetThrReached + ']', e);
 
                     if (failureDetThrReached)
                         LT.warn(log, "Connect timed out (consider increasing 'failureDetectionTimeout' " +
@@ -3041,8 +3046,13 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     X.hasCause(errs, ConnectException.class, HandshakeException.class,
                         SocketTimeoutException.class, HandshakeTimeoutException.class,
                         IgniteSpiOperationTimeoutException.class)) {
-                    U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
-                        "cluster [" + "rmtNode=" + node + ']', errs);
+                    String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
+                        "cluster [" + "rmtNode=" + node + ']';
+
+                    if(enableTroubleshootingLog)
+                        U.error(log, msg, errs);
+                    else
+                        U.warn(log, msg);
 
                     getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
                         "rmtNode=" + node +

From 75c442a8309d554f6894f30f63ed0b964e7a0aa1 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Mon, 26 Jun 2017 15:37:26 +0300
Subject: [PATCH 267/516] Backported IGNITE-5076: improved multi-threaded start
 of nodes.

---
 .../ignite/testframework/junits/GridAbstractTest.java  | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
 mode change 100644 => 100755 modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
old mode 100644
new mode 100755
index ab4d593375130..d28a99109b29a
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -646,12 +646,14 @@ protected final Ignite startGrids(int cnt) throws Exception {
      * @throws Exception If failed.
      */
     protected Ignite startGridsMultiThreaded(int cnt) throws Exception {
-        if (cnt == 1)
-            return startGrids(1);
+        assert cnt > 0 : "Number of grids must be a positive number";
 
-        Ignite ignite = startGridsMultiThreaded(0, cnt);
+        Ignite ignite = startGrids(1);
 
-        checkTopology(cnt);
+        if (cnt > 1) {
+            startGridsMultiThreaded(1, cnt - 1);
+            checkTopology(cnt);
+        }
 
         return ignite;
     }

From 551923e1e5139c484fcb716e7f10907fa71021d9 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 30 Jun 2017 14:38:54 +0300
Subject: [PATCH 268/516] Backported "IGNITE-5424 GridServiceProxy does not
 unwraps exception message from InvocationTargetException."

---
 .../processors/service/GridServiceProxy.java  |  9 ++-
 .../GridServiceProcessorProxySelfTest.java    | 66 +++++++++++++++++++
 2 files changed, 74 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index d16a4c48dcf86..3a40b907f91b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -23,6 +23,7 @@
 import java.io.ObjectOutput;
 import java.io.Serializable;
 import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.util.ArrayList;
@@ -413,7 +414,13 @@ private ServiceProxyCallable(String mtdName, String svcName, Class[] argTypes, O
             if (mtd == null)
                 throw new GridServiceMethodNotFoundException(svcName, mtdName, argTypes);
 
-            return mtd.invoke(svcCtx.service(), args);
+            try {
+                return mtd.invoke(svcCtx.service(), args);
+            }
+            catch (InvocationTargetException e) {
+                // Get error message.
+                throw new IgniteCheckedException(e.getCause().getMessage(), e);
+            }
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
index 7b5abf5d2f90d..7a80fd14a3d54 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
@@ -18,12 +18,15 @@
 package org.apache.ignite.internal.processors.service;
 
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
 
 /**
  * Service proxy test.
@@ -63,6 +66,31 @@ public void testNodeSingletonProxy() throws Exception {
         }
     }
 
+    /**
+     * Unwraps error message from InvocationTargetException.
+     *
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    public void testException() throws Exception {
+        String name = "errorService";
+
+        Ignite ignite = grid(0);
+
+        ignite.services(ignite.cluster().forRemotes()).deployNodeSingleton(name, new ErrorServiceImpl());
+
+        final ErrorService svc = ignite.services().serviceProxy(name, ErrorService.class, false);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                svc.go();
+
+                return null;
+            }
+        }, IgniteException.class, "Test exception");
+
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -353,6 +381,7 @@ protected static class MapServiceImpl<K, V> implements MapService<K, V>, Service
             map.clear();
         }
 
+        /** {@inheritDoc} */
         @Override public int size() {
             return map.size();
         }
@@ -372,4 +401,41 @@ protected static class MapServiceImpl<K, V> implements MapService<K, V>, Service
             X.println("Executing cache service: " + ctx.name());
         }
     }
+
+    /**
+     *
+     */
+    protected interface ErrorService extends Service {
+        /**
+         *
+         */
+        void go() throws Exception;
+    }
+
+    /**
+     *
+     */
+    protected class ErrorServiceImpl implements ErrorService {
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(ServiceContext ctx) throws Exception {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void go() throws Exception {
+            throw new Exception("Test exception");
+        }
+    }
+
+
 }

From 752b1368adbb5c77b5d5caca3c07a72decff5111 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 30 Jun 2017 14:45:18 +0300
Subject: [PATCH 269/516] IGNITE-5473 partial fix: Create ignite
 troubleshooting logger.

---
 .../apache/ignite/IgniteSystemProperties.java |  7 +++++++
 .../tcp/TcpCommunicationSpi.java              | 20 ++++++++++++++-----
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index e9bbf5a10df4c..c4208a76f1a7d 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -135,6 +135,13 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_QUIET = "IGNITE_QUIET";
 
+    /**
+     * Setting this option to {@code true} will enable troubleshooting logger.
+     * Troubleshooting logger makes logging more verbose without enabling debug mode
+     * to provide more detailed logs without performance penalty.
+     */
+    public static final String IGNITE_TROUBLESHOOTING_LOGGER = "IGNITE_TROUBLESHOOTING_LOGGER";
+
     /**
      * Setting to {@code true} enables writing sensitive information in {@code toString()} output.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 9925b3d0c8736..b16842e0d1129 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -333,6 +333,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     private boolean enableForcibleNodeKill = IgniteSystemProperties
         .getBoolean(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
 
+    /** */
+    private boolean enableTroubleshootingLog = IgniteSystemProperties
+        .getBoolean(IgniteSystemProperties.IGNITE_TROUBLESHOOTING_LOGGER);
+
     /** Server listener. */
     private final GridNioServerListener<Message> srvLsnr =
         new GridNioServerListenerAdapter<Message>() {
@@ -2550,9 +2554,10 @@ protected GridCommunicationClient createTcpClient(ClusterNode node) throws Ignit
 
                     boolean failureDetThrReached = timeoutHelper.checkFailureTimeoutReached(e);
 
-                    U.error(log, "Failed to establish connection to a remote node [node=" + node +
-                        ", addr=" + addr + ", connectAttempts=" + connectAttempts +
-                        ", failureDetThrReached" + failureDetThrReached + ']', e);
+                    if (enableTroubleshootingLog)
+                        U.error(log, "Failed to establish connection to a remote node [node=" + node +
+                            ", addr=" + addr + ", connectAttempts=" + connectAttempts +
+                            ", failureDetThrReached=" + failureDetThrReached + ']', e);
 
                     if (failureDetThrReached)
                         LT.warn(log, "Connect timed out (consider increasing 'failureDetectionTimeout' " +
@@ -2602,8 +2607,13 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     X.hasCause(errs, ConnectException.class, HandshakeException.class,
                         SocketTimeoutException.class, HandshakeTimeoutException.class,
                         IgniteSpiOperationTimeoutException.class)) {
-                    U.error(log, "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
-                        "cluster [" + "rmtNode=" + node + ']', errs);
+                    String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
+                        "cluster [" + "rmtNode=" + node + ']';
+
+                    if(enableTroubleshootingLog)
+                        U.error(log, msg, errs);
+                    else
+                        U.warn(log, msg);
 
                     getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
                         "rmtNode=" + node +

From fd3f947a1b49144a5b11aaf3f1a1b7a28ba961fd Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 30 Jun 2017 18:36:50 +0300
Subject: [PATCH 270/516] Fixed service deployment tests.

---
 .../IgniteServiceDynamicCachesSelfTest.java    | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
index c41f2f03c460e..acd2b252d8f9d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
@@ -43,8 +43,8 @@ public class IgniteServiceDynamicCachesSelfTest extends GridCommonAbstractTest {
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
         TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
 
@@ -83,7 +83,7 @@ public void testDeployCalledAfterCacheStart() throws Exception {
 
             final String svcName = "myService";
 
-            svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, "key");
+            svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, primaryKey(ig.cache(cacheName)));
 
             boolean res = GridTestUtils.waitForCondition(new PA() {
                 @Override public boolean apply() {
@@ -125,7 +125,15 @@ public void testDeployCalledBeforeCacheStart() throws Exception {
 
         final String svcName = "myService";
 
-        svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, "key");
+        ig.createCache(ccfg);
+
+        Object key = primaryKey(ig.cache(cacheName));
+
+        ig.destroyCache(cacheName);
+
+        awaitPartitionMapExchange();
+
+        svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, key);
 
         assert svcs.service(svcName) == null;
 
@@ -140,6 +148,8 @@ public void testDeployCalledBeforeCacheStart() throws Exception {
 
             assertTrue("Service was not deployed", res);
 
+            info("stopping cache: " + cacheName);
+
             ig.destroyCache(cacheName);
 
             res = GridTestUtils.waitForCondition(new PA() {

From 6ab152a85f70c05847823f65f8e095ab9eb6b1f7 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 19 Apr 2017 12:46:31 +0300
Subject: [PATCH 271/516] Attempt to fix awaitPartitionMapExchange: wait for
 last exchange completion to avoid races with cache destroy.

(cherry picked from commit d383484)
---
 .../junits/common/GridCommonAbstractTest.java | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 549c486e959aa..12e197124a5da 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -58,6 +58,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
@@ -450,6 +451,27 @@ protected void awaitPartitionMapExchange(boolean waitEvicts,
 
         Set<String> names = new HashSet<>();
 
+        Ignite crd = null;
+
+        for (Ignite g : G.allGrids()) {
+            ClusterNode node = g.cluster().localNode();
+
+            if (crd == null || node.order() < crd.cluster().localNode().order()) {
+                crd = g;
+
+                if (node.order() == 1)
+                    break;
+            }
+        }
+
+        if (crd == null)
+            return;
+
+        AffinityTopologyVersion waitTopVer = ((IgniteKernal)crd).context().discovery().topologyVersionEx();
+
+        if (waitTopVer.topologyVersion() <= 0)
+            waitTopVer = new AffinityTopologyVersion(1, 0);
+
         for (Ignite g : G.allGrids()) {
             if (nodes != null && !nodes.contains(g.cluster().localNode()))
                 continue;
@@ -466,6 +488,19 @@ protected void awaitPartitionMapExchange(boolean waitEvicts,
             else
                 startTime = g0.context().discovery().gridStartTime();
 
+            IgniteInternalFuture<?> exchFut =
+                g0.context().cache().context().exchange().affinityReadyFuture(waitTopVer);
+
+            if (exchFut != null && !exchFut.isDone()) {
+                try {
+                    exchFut.get(timeout);
+                }
+                catch (IgniteCheckedException e) {
+                    log.error("Failed to wait for exchange [topVer=" + waitTopVer +
+                        ", node=" + g0.name() + ']', e);
+                }
+            }
+
             for (IgniteCacheProxy<?, ?> c : g0.context().cache().jcaches()) {
                 CacheConfiguration cfg = c.context().config();
 

From 1525c6cf2cb015289392eb54fec4029e9b53b438 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 30 Jun 2017 18:36:50 +0300
Subject: [PATCH 272/516] Fixed service deployment tests.

---
 .../IgniteServiceDynamicCachesSelfTest.java    | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
index c41f2f03c460e..acd2b252d8f9d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
@@ -43,8 +43,8 @@ public class IgniteServiceDynamicCachesSelfTest extends GridCommonAbstractTest {
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
         TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
 
@@ -83,7 +83,7 @@ public void testDeployCalledAfterCacheStart() throws Exception {
 
             final String svcName = "myService";
 
-            svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, "key");
+            svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, primaryKey(ig.cache(cacheName)));
 
             boolean res = GridTestUtils.waitForCondition(new PA() {
                 @Override public boolean apply() {
@@ -125,7 +125,15 @@ public void testDeployCalledBeforeCacheStart() throws Exception {
 
         final String svcName = "myService";
 
-        svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, "key");
+        ig.createCache(ccfg);
+
+        Object key = primaryKey(ig.cache(cacheName));
+
+        ig.destroyCache(cacheName);
+
+        awaitPartitionMapExchange();
+
+        svcs.deployKeyAffinitySingleton(svcName, new TestService(), cacheName, key);
 
         assert svcs.service(svcName) == null;
 
@@ -140,6 +148,8 @@ public void testDeployCalledBeforeCacheStart() throws Exception {
 
             assertTrue("Service was not deployed", res);
 
+            info("stopping cache: " + cacheName);
+
             ig.destroyCache(cacheName);
 
             res = GridTestUtils.waitForCondition(new PA() {

From a24ca24dd7aaa34707d74a4e660d769d3d5b0ed8 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Wed, 19 Apr 2017 12:46:31 +0300
Subject: [PATCH 273/516] Attempt to fix awaitPartitionMapExchange: wait for
 last exchange completion to avoid races with cache destroy.

(cherry picked from commit d383484)
---
 .../junits/common/GridCommonAbstractTest.java | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 90fabd95a1b47..6616daf297388 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -58,6 +58,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
@@ -450,6 +451,27 @@ protected void awaitPartitionMapExchange(boolean waitEvicts,
 
         Set<String> names = new HashSet<>();
 
+        Ignite crd = null;
+
+        for (Ignite g : G.allGrids()) {
+            ClusterNode node = g.cluster().localNode();
+
+            if (crd == null || node.order() < crd.cluster().localNode().order()) {
+                crd = g;
+
+                if (node.order() == 1)
+                    break;
+            }
+        }
+
+        if (crd == null)
+            return;
+
+        AffinityTopologyVersion waitTopVer = ((IgniteKernal)crd).context().discovery().topologyVersionEx();
+
+        if (waitTopVer.topologyVersion() <= 0)
+            waitTopVer = new AffinityTopologyVersion(1, 0);
+
         for (Ignite g : G.allGrids()) {
             if (nodes != null && !nodes.contains(g.cluster().localNode()))
                 continue;
@@ -466,6 +488,19 @@ protected void awaitPartitionMapExchange(boolean waitEvicts,
             else
                 startTime = g0.context().discovery().gridStartTime();
 
+            IgniteInternalFuture<?> exchFut =
+                g0.context().cache().context().exchange().affinityReadyFuture(waitTopVer);
+
+            if (exchFut != null && !exchFut.isDone()) {
+                try {
+                    exchFut.get(timeout);
+                }
+                catch (IgniteCheckedException e) {
+                    log.error("Failed to wait for exchange [topVer=" + waitTopVer +
+                        ", node=" + g0.name() + ']', e);
+                }
+            }
+
             for (IgniteCacheProxy<?, ?> c : g0.context().cache().jcaches()) {
                 CacheConfiguration cfg = c.context().config();
 

From 05fac7e8a0bf9e08cd758bed7bd35ec85b914592 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Wed, 19 Apr 2017 14:01:21 +0300
Subject: [PATCH 274/516] Backported IGNITE-4925 Fix
 IgniteCacheBinaryObjectsScanSelfTest.testScanNoClasses - Fixes #1750.

(cherry picked from commit b47f29d)
---
 .../cache/IgniteCacheBinaryObjectsScanSelfTest.java   | 11 +++++++++--
 .../cache/IgniteCacheEntryListenerAbstractTest.java   |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanSelfTest.java
index 77438821630b1..69cda8538e81c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanSelfTest.java
@@ -17,10 +17,12 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.ArrayList;
 import java.util.List;
 import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -127,9 +129,14 @@ public void testScanNoClasses() throws Exception {
             assertEquals(PERSON_CLS_NAME, entry.getValue().getClass().getName());
         }
 
-        entries = cache.query(new ScanQuery<>(1)).getAll();
+        entries = new ArrayList<>();
 
-        assertFalse(entries.isEmpty());
+        int partCnt = client.affinity("testCache").partitions();
+
+        for (int i = 0; i < partCnt; i++)
+            entries.addAll(cache.query(new ScanQuery<>(i)).getAll());
+
+        assertEquals(100, entries.size());
 
         for (Cache.Entry<Object, Object> entry : entries) {
             assertEquals(PERSON_KEY_CLS_NAME, entry.getKey().getClass().getName());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java
index 1f58765b85364..1772ef95cc938 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java
@@ -339,7 +339,7 @@ private void awaitLatch() {
                 if (!eagerTtl()) {
                     U.sleep(1100);
 
-                    assertNull(primaryCache(key, cache.getName()).get(key(key)));
+                    assertNull(primaryCache(key(key), cache.getName()).get(key(key)));
 
                     evtsLatch.await(5000, MILLISECONDS);
 

From ef0a874ceb5c8bfa53e16337f6fd1699afaf2a39 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 30 Jun 2017 20:39:01 +0300
Subject: [PATCH 275/516] Fixed
 CacheSerializableTransactionsTest#testTxConflictRemoveWithOldValue test.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../ignite/internal/processors/cache/GridCacheAdapter.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 220c192f21c46..cde6309d3f9ff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3011,7 +3011,7 @@ protected boolean remove0(final K key, final CacheEntryPredicate filter) throws
                     Collections.singletonList(key),
                     /*retval*/false,
                     filter,
-                    /*singleRmv*/true).get().success();
+                    /*singleRmv*/false).get().success();
             }
 
             @Override public String toString() {

From 4dce965ea86374cba7265cb5d22e975aeac7d480 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 30 Jun 2017 21:36:02 +0300
Subject: [PATCH 276/516] Fixed org.jsr107.tck.PutTest tests.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../internal/processors/cache/GridCacheAdapter.java   | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index cde6309d3f9ff..7c5a54dfcb852 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -642,7 +642,7 @@ public void onKernalStop() {
 
         return (IgniteInternalFuture)getAsync(
             key,
-            /*force primary*/false,
+            /*force primary*/ !ctx.config().isReadFromBackup(),
             /*skip tx*/false,
             /*subj id*/null,
             /*task name*/null,
@@ -669,7 +669,7 @@ public void onKernalStop() {
 
         return getAllAsync(
             keys,
-            /*force primary*/false,
+            /*force primary*/ !ctx.config().isReadFromBackup(),
             /*skip tx*/false,
             /*subj id*/null,
             /*task name*/null,
@@ -1137,7 +1137,7 @@ public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, bool
                     execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
 
                     for (int i = 1; i < jobs.size(); i++)
-                        execSvc.submit(jobs.get(i));
+                        execSvc.execute(jobs.get(i));
                 }
 
                 jobs.get(0).run();
@@ -2715,6 +2715,8 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
 
     /** {@inheritDoc} */
     @Override public void putAll(@Nullable final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        A.notNull(m, "map");
+
         if (F.isEmpty(m))
             return;
 
@@ -3000,6 +3002,7 @@ public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws
 
     /**
      * @param key Key.
+     * @param filter Filter.
      * @return {@code True} if entry was removed.
      * @throws IgniteCheckedException If failed.
      */
@@ -3011,7 +3014,7 @@ protected boolean remove0(final K key, final CacheEntryPredicate filter) throws
                     Collections.singletonList(key),
                     /*retval*/false,
                     filter,
-                    /*singleRmv*/false).get().success();
+                    /*singleRmv*/filter == null).get().success();
             }
 
             @Override public String toString() {

From 50887fed508e03a8b7df32569afb6d84ab3eb670 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 4 Jul 2017 20:01:01 +0300
Subject: [PATCH 277/516] IGNITE-5663: ODBC: Closing cursor do not reset
 prepared statement anymore

---
 .../cpp/odbc-test/src/queries_test.cpp        | 62 +++++++++++++++++++
 modules/platforms/cpp/odbc/src/statement.cpp  |  3 -
 2 files changed, 62 insertions(+), 3 deletions(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 9aca77de784ed..f0e7c76133ed9 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -1754,4 +1754,66 @@ BOOST_AUTO_TEST_CASE(TestParamsNum)
     CheckParamsNum("INSERT INTO TestType(_key, strField) VALUES(?, ?)", 2);
 }
 
+BOOST_AUTO_TEST_CASE(TestExecuteAfterCursorClose)
+{
+    TestType in(1, 2, 3, 4, "5", 6.0f, 7.0, true, Guid(8, 9), BinaryUtils::MakeDateGmt(1987, 6, 5),
+        BinaryUtils::MakeTimestampGmt(1998, 12, 27, 1, 2, 3, 456));
+
+    cache1.Put(1, in);
+
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+
+    int64_t key = 0;
+    char strField[1024] = { 0 };
+    SQLLEN strFieldLen = 0;
+
+    // Binding columns.
+    SQLRETURN ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    // Binding columns.
+    ret = SQLBindCol(stmt, 2, SQL_C_CHAR, &strField, sizeof(strField), &strFieldLen);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    // Just selecting everything to make sure everything is OK
+    SQLCHAR selectReq[] = "SELECT _key, strField FROM TestType";
+
+    ret = SQLPrepare(stmt, selectReq, sizeof(selectReq));
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLExecute(stmt);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLFreeStmt(stmt, SQL_CLOSE);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLExecute(stmt);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    ret = SQLFetch(stmt);
+
+    if (!SQL_SUCCEEDED(ret))
+        BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+    BOOST_CHECK_EQUAL(key, 1);
+
+    BOOST_CHECK_EQUAL(std::string(strField, strFieldLen), "5");
+
+    ret = SQLFetch(stmt);
+
+    BOOST_CHECK_EQUAL(ret, SQL_NO_DATA);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp
index 6154f9126568e..78b8a1ef8a2da 100644
--- a/modules/platforms/cpp/odbc/src/statement.cpp
+++ b/modules/platforms/cpp/odbc/src/statement.cpp
@@ -684,9 +684,6 @@ namespace ignite
 
             SqlResult result = currentQuery->Close();
 
-            if (result == SQL_RESULT_SUCCESS)
-                currentQuery.reset();
-
             return result;
         }
 

From da290cee855ef45a90ad539515e039f2826a6c00 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Wed, 5 Jul 2017 13:21:12 +0300
Subject: [PATCH 278/516] IGNITE-5663: Fix for test

---
 modules/platforms/cpp/odbc-test/src/queries_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index f0e7c76133ed9..4c0284f978ca9 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -1761,7 +1761,7 @@ BOOST_AUTO_TEST_CASE(TestExecuteAfterCursorClose)
 
     cache1.Put(1, in);
 
-    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+    Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
 
     int64_t key = 0;
     char strField[1024] = { 0 };

From 3536a58982e4c264bb72b2ccc1953049d2b5c67f Mon Sep 17 00:00:00 2001
From: Alexey Kukushkin <AlexeyKukushkin@yahoo.com>
Date: Wed, 5 Jul 2017 19:36:41 +0300
Subject: [PATCH 279/516] IGNITE-4901 Decrease logging level for DataStremer
 retry

---
 .../datastreamer/DataStreamProcessor.java     |   4 +-
 .../datastreamer/DataStreamerImpl.java        |  14 +-
 .../DataStreamerImplSelfTest.java             | 121 +++++++++++++++++-
 3 files changed, 131 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index 6f35a52d5b673..dec228d520ec1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -21,10 +21,10 @@
 import java.util.UUID;
 import java.util.concurrent.DelayQueue;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.communication.GridIoManager;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.deployment.GridDeployment;
@@ -336,7 +336,7 @@ private void localUpdate(final UUID nodeId,
                 AffinityTopologyVersion topVer = fut.topologyVersion();
 
                 if (!allowOverwrite && !topVer.equals(req.topologyVersion())) {
-                    Exception err = new IgniteCheckedException(
+                    Exception err = new ClusterTopologyCheckedException(
                         "DataStreamer will retry data transfer at stable topology " +
                             "[reqTop=" + req.topologyVersion() + ", topVer=" + topVer + ", node=remote]");
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index 515314eb08728..8978fbf34dffe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -1774,10 +1774,16 @@ void onResponse(DataStreamerResponse res, UUID nodeId) {
                 try {
                     GridPeerDeployAware jobPda0 = jobPda;
 
-                    err = new IgniteCheckedException("DataStreamer request failed [node=" + nodeId + "]",
-                        (Throwable)U.unmarshal(ctx,
-                            errBytes,
-                            U.resolveClassLoader(jobPda0 != null ? jobPda0.classLoader() : null, ctx.config())));
+                    final Throwable cause = U.unmarshal(
+                        ctx,
+                        errBytes,
+                        U.resolveClassLoader(jobPda0 != null ? jobPda0.classLoader() : null, ctx.config()));
+
+                    final String msg = "DataStreamer request failed [node=" + nodeId + "]";
+
+                    err = cause instanceof ClusterTopologyCheckedException ?
+                        new ClusterTopologyCheckedException(msg, cause) :
+                        new IgniteCheckedException(msg, cause);
                 }
                 catch (IgniteCheckedException e) {
                     f.onDone(null, new IgniteCheckedException("Failed to unmarshal response.", e));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java
index a6a9f5488cefc..b26ccf24526eb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.datastreamer;
 
 import java.io.Serializable;
+import java.io.StringWriter;
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.Callable;
@@ -26,17 +27,29 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.cache.CacheServerNotFoundException;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.log4j.Appender;
+import org.apache.log4j.Logger;
+import org.apache.log4j.SimpleLayout;
+import org.apache.log4j.WriterAppender;
 
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
@@ -51,12 +64,18 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest {
     /** Number of keys to load via data streamer. */
     private static final int KEYS_COUNT = 1000;
 
+    /** Next nodes after MAX_CACHE_COUNT start without cache */
+    private static final int MAX_CACHE_COUNT = 4;
+
     /** Started grid counter. */
     private static int cnt;
 
     /** No nodes filter. */
     private static volatile boolean noNodesFilter;
 
+    /** Indicates whether we need to make the topology stale */
+    private static boolean needStaleTop = false;
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         super.afterTest();
@@ -73,8 +92,9 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest {
 
         cfg.setDiscoverySpi(discoSpi);
 
-        // Forth node goes without cache.
-        if (cnt < 4)
+        cfg.setCommunicationSpi(new StaleTopologyCommunicationSpi());
+
+        if (cnt < MAX_CACHE_COUNT)
             cfg.setCacheConfiguration(cacheConfiguration());
 
         cnt++;
@@ -232,6 +252,44 @@ public void testNoDataNodesOnFlush() throws Exception {
         }
     }
 
+    /**
+     * Cluster topology mismatch shall result in DataStreamer retrying cache update with the latest topology and
+     * no error logged to the console.
+     *
+     * @throws Exception if failed
+     */
+    public void testRetryWhenTopologyMismatch() throws Exception {
+        final int KEY = 1;
+        final String VAL = "1";
+
+        cnt = 0;
+
+        StringWriter logWriter = new StringWriter();
+        Appender logAppender = new WriterAppender(new SimpleLayout(), logWriter);
+
+        Logger.getRootLogger().addAppender(logAppender);
+
+        startGrids(MAX_CACHE_COUNT - 1); // cache-enabled nodes
+
+        try (Ignite ignite = startGrid(MAX_CACHE_COUNT);
+             IgniteDataStreamer<Integer, String> streamer = ignite.dataStreamer(null)) {
+
+            needStaleTop = true; // simulate stale topology for the next action
+
+            streamer.addData(KEY, VAL);
+        } finally {
+            needStaleTop = false;
+
+            logWriter.flush();
+
+            Logger.getRootLogger().removeAppender(logAppender);
+
+            logAppender.close();
+        }
+
+        assertFalse(logWriter.toString().contains("DataStreamer will retry data transfer at stable topology"));
+    }
+
     /**
      * Gets cache configuration.
      *
@@ -284,4 +342,63 @@ public Integer val() {
             return obj instanceof TestObject && ((TestObject)obj).val == val;
         }
     }
+
+    /**
+     * Simulate stale (not up-to-date) topology
+     */
+    private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+            // Send stale topology only in the first request to avoid indefinitely getting failures.
+            if (needStaleTop) {
+                if (msg instanceof GridIoMessage) {
+                    GridIoMessage ioMsg = (GridIoMessage)msg;
+
+                    Message appMsg = ioMsg.message();
+
+                    if (appMsg != null && appMsg instanceof DataStreamerRequest) {
+                        DataStreamerRequest req = (DataStreamerRequest)appMsg;
+
+                        AffinityTopologyVersion validTop = req.topologyVersion();
+
+                        // Simulate situation when a node did not receive the latest "node joined" topology update causing
+                        // topology mismatch
+                        AffinityTopologyVersion staleTop = new AffinityTopologyVersion(
+                            validTop.topologyVersion() - 1,
+                            validTop.minorTopologyVersion());
+
+                        appMsg = new DataStreamerRequest(
+                            req.requestId(),
+                            req.responseTopicBytes(),
+                            req.cacheName(),
+                            req.updaterBytes(),
+                            req.entries(),
+                            req.ignoreDeploymentOwnership(),
+                            req.skipStore(),
+                            req.keepBinary(),
+                            req.deploymentMode(),
+                            req.sampleClassName(),
+                            req.userVersion(),
+                            req.participants(),
+                            req.classLoaderId(),
+                            req.forceLocalDeployment(),
+                            staleTop);
+
+                        msg = new GridIoMessage(
+                            GridTestUtils.<Byte>getFieldValue(ioMsg, "plc"),
+                            GridTestUtils.getFieldValue(ioMsg, "topic"),
+                            GridTestUtils.<Integer>getFieldValue(ioMsg, "topicOrd"),
+                            appMsg,
+                            GridTestUtils.<Boolean>getFieldValue(ioMsg, "ordered"),
+                            ioMsg.timeout(),
+                            ioMsg.skipOnTimeout());
+
+                        needStaleTop = false;
+                    }
+                }
+            }
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
 }
\ No newline at end of file

From 6d3a3ff2d99697882232070e715928336a9180cd Mon Sep 17 00:00:00 2001
From: Alexey Kukushkin <AlexeyKukushkin@yahoo.com>
Date: Wed, 5 Jul 2017 20:05:02 +0300
Subject: [PATCH 280/516] Fixed merge conflicts

---
 .../cache/GridCachePartitionExchangeManager.java    |  2 +-
 .../spi/communication/tcp/TcpCommunicationSpi.java  | 13 -------------
 2 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 8ddbb59b540f3..c62ffd2aec79c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -269,7 +269,7 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                                 exchFut = exchangeFuture(exchId, evt, cache, null, msg);
                             }
                         }
-                        elseif (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                        else if (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
                             exchangeFuture(msg.exchangeId(), null, null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
                     }
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index f96cd1d79f538..0d80f4778df36 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3504,19 +3504,6 @@ private static class HandshakeException extends IgniteCheckedException {
         }
     }
 
-    /** Internal exception class for proper timeout handling. */
-    private static class HandshakeException extends IgniteCheckedException {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /**
-         * @param msg Error message.
-         */
-        HandshakeException(String msg) {
-            super(msg);
-        }
-    }
-
     /** Internal exception class for proper timeout handling. */
     private static class HandshakeTimeoutException extends IgniteCheckedException {
         /** */

From acfc400b22738fa46397d392f88d49614e687969 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Wed, 5 Jul 2017 20:42:48 +0300
Subject: [PATCH 281/516] Merge branch 'ignite-1.7.12' into ignite-1.9.4

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../spi/communication/tcp/TcpCommunicationSpi.java  | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index ed672e2fe4730..1776b01ff8da5 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3504,19 +3504,6 @@ private static class HandshakeException extends IgniteCheckedException {
         }
     }
 
-    /** Internal exception class for proper timeout handling. */
-    private static class HandshakeException extends IgniteCheckedException {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /**
-         * @param msg Error message.
-         */
-        HandshakeException(String msg) {
-            super(msg);
-        }
-    }
-
     /** Internal exception class for proper timeout handling. */
     private static class HandshakeTimeoutException extends IgniteCheckedException {
         /** */

From 8dea19ba41bb9eda16f47933b2c46a081116d5f7 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 6 Jul 2017 12:02:07 +0300
Subject: [PATCH 282/516] Minor fix.

---
 .../processors/cache/GridCachePartitionExchangeManager.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 8ddbb59b540f3..c62ffd2aec79c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -269,7 +269,7 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                                 exchFut = exchangeFuture(exchId, evt, cache, null, msg);
                             }
                         }
-                        elseif (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                        else if (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
                             exchangeFuture(msg.exchangeId(), null, null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
                     }
                 }

From f208f434f944196d531a1b51066dfe8d6394d739 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 6 Jul 2017 15:17:50 +0300
Subject: [PATCH 283/516] Test fixed "IGNITE-5390: Bug in
 IgniteCacheTxStoreSessionWriteBehindCoalescingTest."

---
 .../cache/IgniteCacheAbstractTest.java        | 17 +++++++++
 ...ientWriteBehindStoreNonCoalescingTest.java | 38 +++++++++++--------
 2 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractTest.java
index 45b6e9d07e757..f9ae6c24dfdf9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractTest.java
@@ -152,6 +152,9 @@ protected CacheConfiguration cacheConfiguration(String gridName) throws Exceptio
             cfg.setReadThrough(true);
             cfg.setWriteThrough(true);
             cfg.setLoadPreviousValue(true);
+
+            cfg.setWriteBehindEnabled(writeBehindEnabled());
+            cfg.setWriteBehindCoalescing(writeBehindCoalescing());
         }
 
         if (cacheMode() == PARTITIONED)
@@ -167,6 +170,20 @@ protected Factory<CacheStore> cacheStoreFactory() {
         return null;
     }
 
+    /**
+     * @return write behind enabled flag.
+     */
+    protected boolean writeBehindEnabled() {
+        return false;
+    }
+
+    /**
+     * @return write behind coalescing flag.
+     */
+    protected boolean writeBehindCoalescing() {
+        return true;
+    }
+
     /**
      * @return Cache loader factory.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java
index 8ea109dd2edfd..297a1cca8e714 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java
@@ -19,9 +19,8 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashSet;
 import java.util.Map;
-import java.util.Set;
+import java.util.Random;
 import javax.cache.Cache;
 import javax.cache.configuration.Factory;
 import javax.cache.processor.EntryProcessor;
@@ -77,6 +76,18 @@ public class IgnteCacheClientWriteBehindStoreNonCoalescingTest extends IgniteCac
         return new TestIncrementStoreFactory();
     }
 
+    /** {@inheritDoc} */
+    @Override protected boolean writeBehindEnabled() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean writeBehindCoalescing() {
+        return false;
+    }
+
+    private static Random rnd = new Random();
+
     /**
      * @throws Exception If failed.
      */
@@ -88,35 +99,30 @@ public void testNonCoalescingIncrementing() throws Exception {
         assertEquals(cache.getConfiguration(CacheConfiguration.class).getCacheStoreFactory().getClass(),
             TestIncrementStoreFactory.class);
 
-        Set<Integer> keys = new HashSet<>();
-
-        for (int i = 0; i < 1000; i++) {
-            keys.add(i);
-
+        for (int i = 0; i < CacheConfiguration.DFLT_WRITE_BEHIND_FLUSH_SIZE * 2; i++) {
             cache.put(i, i);
         }
 
         Collection<IgniteFuture<?>> futs = new ArrayList<>();
 
-        for (int i = 0; i < 100; i++)
-            futs.add(updateKeys(cache, keys));
+        for (int i = 0; i < 1000; i++)
+            futs.add(updateKey(cache));
 
         for (IgniteFuture<?> fut : futs)
             fut.get();
     }
 
     /**
-     * Update specified keys in async mode.
+     * Update random key in async mode.
      *
      * @param cache Cache to use.
-     * @param keys Keys to update.
      * @return IgniteFuture.
      */
-    private IgniteFuture<?>  updateKeys(IgniteCache<Integer, Integer> cache, Set<Integer> keys) {
+    private IgniteFuture<?> updateKey(IgniteCache<Integer, Integer> cache) {
         IgniteCache asyncCache = cache.withAsync();
 
         // Using EntryProcessor.invokeAll to increment every value in place.
-        asyncCache.invokeAll(keys, new EntryProcessor<Integer, Integer, Object>() {
+        asyncCache.invoke(rnd.nextInt(100), new EntryProcessor<Integer, Integer, Object>() {
             @Override public Object process(MutableEntry<Integer, Integer> entry, Object... arguments)
                 throws EntryProcessorException {
                 entry.setValue(entry.getValue() + 1);
@@ -155,10 +161,10 @@ public static class TestIncrementStore extends CacheStoreAdapter<Object, Object>
 
         /** {@inheritDoc} */
         @Override public void write(Cache.Entry<? extends Object, ? extends Object> entry) {
-            Object oldValue = storeMap.put(entry.getKey(), entry.getValue());
+            Object oldVal = storeMap.put(entry.getKey(), entry.getValue());
 
-            if (oldValue instanceof Integer && entry.getValue() instanceof Integer) {
-                Integer oldInt = (Integer)oldValue;
+            if (oldVal != null) {
+                Integer oldInt = (Integer)oldVal;
                 Integer newInt = (Integer)entry.getValue();
 
                 assertTrue(

From 355a5283559c885f57c4557bba2c6d9170a9b5fc Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Fri, 30 Jun 2017 20:23:55 +0300
Subject: [PATCH 284/516] IGNITE-5554 ServiceProcessor may process failed
 reassignments in timeout thread

---
 .../service/GridServiceProcessor.java         | 21 +++++++++----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 20bcff7c780d3..c452da31086d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1581,16 +1581,11 @@ private void onDeployment(final GridServiceDeployment dep, final AffinityTopolog
                 }
 
                 @Override public void onTimeout() {
-                    if (!busyLock.enterBusy())
-                        return;
-
-                    try {
-                        // Try again.
-                        onDeployment(dep, topVer);
-                    }
-                    finally {
-                        busyLock.leaveBusy();
-                    }
+                    depExe.execute(new DepRunnable() {
+                        @Override public void run0() {
+                            onDeployment(dep, topVer);
+                        }
+                    });
                 }
             });
         }
@@ -1777,7 +1772,11 @@ private void onReassignmentFailed(final AffinityTopologyVersion topVer,
                         }
 
                         @Override public void onTimeout() {
-                            onReassignmentFailed(topVer, retries);
+                            depExe.execute(new Runnable() {
+                                public void run() {
+                                    onReassignmentFailed(topVer, retries);
+                                }
+                            });
                         }
                     });
                 }

From 92aa7c6e3c0d9b5cc68002433861b175d54f9421 Mon Sep 17 00:00:00 2001
From: agura <agura@gridgain.com>
Date: Tue, 4 Jul 2017 16:56:40 +0300
Subject: [PATCH 285/516] ignite-5685 JDBC prepared statement shouldn't clear
 parameters after execution

---
 .../jdbc2/JdbcPreparedStatementSelfTest.java  | 35 +++++++++++++++++++
 .../jdbc/JdbcPreparedStatementSelfTest.java   | 35 +++++++++++++++++++
 .../internal/jdbc/JdbcPreparedStatement.java  |  6 +---
 .../internal/jdbc2/JdbcPreparedStatement.java |  6 ++--
 4 files changed, 73 insertions(+), 9 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
index ea586b297a579..56c248817e12a 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java
@@ -137,6 +137,41 @@ public class JdbcPreparedStatementSelfTest extends GridCommonAbstractTest {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRepeatableUsage() throws Exception {
+        stmt = conn.prepareStatement("select * from TestObject where id = ?");
+
+        stmt.setInt(1, 1);
+
+        ResultSet rs = stmt.executeQuery();
+
+        int cnt = 0;
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assertEquals(1, rs.getInt(1));
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+
+        cnt = 0;
+
+        rs = stmt.executeQuery();
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assertEquals(1, rs.getInt(1));
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java
index 0c701ef3e7ce3..cd8648833c3d1 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java
@@ -155,6 +155,41 @@ public class JdbcPreparedStatementSelfTest extends GridCommonAbstractTest {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRepeatableUsage() throws Exception {
+        stmt = conn.prepareStatement("select * from TestObject where id = ?");
+
+        stmt.setInt(1, 1);
+
+        ResultSet rs = stmt.executeQuery();
+
+        int cnt = 0;
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assertEquals(1, rs.getInt(1));
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+
+        cnt = 0;
+
+        rs = stmt.executeQuery();
+
+        while (rs.next()) {
+            if (cnt == 0)
+                assertEquals(1, rs.getInt(1));
+
+            cnt++;
+        }
+
+        assertEquals(1, cnt);
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java
index 7e5358becab93..93cda1e4768cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java
@@ -69,11 +69,7 @@ public class JdbcPreparedStatement extends JdbcStatement implements PreparedStat
 
     /** {@inheritDoc} */
     @Override public ResultSet executeQuery() throws SQLException {
-        ResultSet rs = executeQuery(sql);
-
-        args = null;
-
-        return rs;
+        return executeQuery(sql);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
index a99f24c8633d7..72375cd939a88 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
@@ -50,11 +50,9 @@ public class JdbcPreparedStatement extends JdbcStatement implements PreparedStat
 
     /** {@inheritDoc} */
     @Override public ResultSet executeQuery() throws SQLException {
-        ResultSet rs = executeQuery(sql);
-
-        args = null;
+        ensureNotClosed();
 
-        return rs;
+        return executeQuery(sql);
     }
 
     /** {@inheritDoc} */

From 9165a0f93b5173b543cc6b4fad5fde37bd215f91 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 7 Jul 2017 15:35:33 +0300
Subject: [PATCH 286/516] ignite-5562: assert statements were changed to the
 'if' blocks

---
 .../spi/discovery/tcp/internal/TcpDiscoveryStatistics.java  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
index a69dbd9aa89aa..42d2ae6f8617e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
@@ -437,7 +437,8 @@ public synchronized String maxProcessingTimeMessageClass() {
      * @param initTime Time socket was initialized in.
      */
     public synchronized void onServerSocketInitialized(long initTime) {
-        assert initTime >= 0;
+        if (initTime < 0)
+            initTime = 0;
 
         if (maxSrvSockInitTime < initTime)
             maxSrvSockInitTime = initTime;
@@ -449,7 +450,8 @@ public synchronized void onServerSocketInitialized(long initTime) {
      * @param initTime Time socket was initialized in.
      */
     public synchronized void onClientSocketInitialized(long initTime) {
-        assert initTime >= 0;
+        if (initTime < 0)
+            initTime = 0;
 
         clientSockCreatedCnt++;
 

From d9fc20a61d5ac0a6e63b26faa7fa0af753b2fa06 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dgovorukhin@gridgain.com>
Date: Fri, 7 Apr 2017 14:28:22 +0300
Subject: [PATCH 287/516] IGNITE-4889 - Changed Hibernate integration to use
 custom keys

(cherry picked from commit 6b62a20)
---
 .../HibernateAccessStrategyAdapter.java       |   8 +-
 .../cache/hibernate/HibernateCacheProxy.java  | 818 ++++++++++++++++++
 .../hibernate/HibernateCollectionRegion.java  |   3 +-
 .../hibernate/HibernateEntityRegion.java      |   3 +-
 .../hibernate/HibernateGeneralDataRegion.java |   3 +-
 .../hibernate/HibernateKeyTransformer.java    |  29 +
 .../cache/hibernate/HibernateKeyWrapper.java  |  72 ++
 .../hibernate/HibernateNaturalIdRegion.java   |   3 +-
 .../HibernateNonStrictAccessStrategy.java     |   5 +-
 .../HibernateQueryResultsRegion.java          |   3 +-
 .../HibernateReadOnlyAccessStrategy.java      |   3 +-
 .../HibernateReadWriteAccessStrategy.java     |   3 +-
 .../cache/hibernate/HibernateRegion.java      |   6 +-
 .../hibernate/HibernateRegionFactory.java     |  29 +-
 .../hibernate/HibernateTimestampsRegion.java  |   3 +-
 .../HibernateTransactionalAccessStrategy.java |   2 +-
 .../HibernateTransactionalDataRegion.java     |   3 +-
 .../IgniteBinaryHibernateTestSuite.java       |  37 +
 18 files changed, 1000 insertions(+), 33 deletions(-)
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
 create mode 100644 modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java

diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
index 27734d9ad9ca5..f6c1d0e15eb43 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
@@ -92,7 +92,7 @@
  */
 public abstract class HibernateAccessStrategyAdapter {
     /** */
-    protected final IgniteInternalCache<Object, Object> cache;
+    protected final HibernateCacheProxy cache;
 
     /** Grid. */
     protected final Ignite ignite;
@@ -104,7 +104,7 @@ public abstract class HibernateAccessStrategyAdapter {
      * @param ignite Grid.
      * @param cache Cache.
      */
-    protected HibernateAccessStrategyAdapter(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    protected HibernateAccessStrategyAdapter(Ignite ignite, HibernateCacheProxy cache) {
         this.cache = cache;
         this.ignite = ignite;
 
@@ -292,8 +292,10 @@ protected final void removeAll() throws CacheException {
      * @param key Key.
      * @throws CacheException If failed.
      */
-    static void evict(Ignite ignite, IgniteInternalCache<Object,Object> cache, Object key) throws CacheException {
+    static void evict(Ignite ignite, HibernateCacheProxy cache, Object key) throws CacheException {
         try {
+            key = cache.keyTransformer().transform(key);
+
             ignite.compute(ignite.cluster()).call(new ClearKeyCallable(key, cache.name()));
         }
         catch (IgniteException e) {
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
new file mode 100644
index 0000000000000..871c4a100ac6f
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
@@ -0,0 +1,818 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorResult;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheEntry;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.mxbean.CacheMetricsMXBean;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Hibernate cache proxy.
+ */
+public class HibernateCacheProxy implements IgniteInternalCache<Object, Object> {
+    /** Delegate. */
+    private final IgniteInternalCache<Object, Object> delegate;
+
+    /** Transformer. */
+    private final HibernateKeyTransformer keyTransformer;
+
+    /**
+     * @param delegate Delegate.
+     * @param keyTransformer Key keyTransformer.
+     */
+    HibernateCacheProxy(
+        IgniteInternalCache<Object, Object> delegate,
+        HibernateKeyTransformer keyTransformer
+    ) {
+        assert delegate != null;
+        assert keyTransformer != null;
+
+        this.delegate = delegate;
+        this.keyTransformer = keyTransformer;
+    }
+
+    /**
+     * @return HibernateKeyTransformer
+     */
+    HibernateKeyTransformer keyTransformer(){
+        return keyTransformer;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String name() {
+        return delegate.name();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean skipStore() {
+        return delegate.skipStore();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache setSkipStore(boolean skipStore) {
+        return delegate.setSkipStore(skipStore);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean containsKey(Object key) {
+        return delegate.containsKey(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> containsKeyAsync(Object key) {
+        return delegate.containsKeyAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void promoteAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.promoteAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public long overflowSize() throws IgniteCheckedException {
+        return delegate.overflowSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long swapSize() throws IgniteCheckedException {
+        return delegate.swapSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long swapKeys() throws IgniteCheckedException {
+        return delegate.swapKeys();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean containsKeys(Collection keys) {
+        return delegate.containsKey(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> containsKeysAsync(Collection keys) {
+        return delegate.containsKeysAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object localPeek(
+        Object key,
+        CachePeekMode[] peekModes,
+        @Nullable IgniteCacheExpiryPolicy plc
+    ) throws IgniteCheckedException {
+        return delegate.localPeek(keyTransformer.transform(key), peekModes, plc);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable<Cache.Entry<Object, Object>> localEntries(
+        CachePeekMode[] peekModes
+    ) throws IgniteCheckedException {
+        return delegate.localEntries(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object get(Object key) throws IgniteCheckedException {
+        return delegate.get(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public CacheEntry getEntry(Object key) throws IgniteCheckedException {
+        return delegate.getEntry(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAsync(Object key) {
+        return delegate.getAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<CacheEntry<Object, Object>> getEntryAsync(Object key) {
+        return delegate.getEntryAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map getAll(@Nullable Collection keys) throws IgniteCheckedException {
+        return delegate.getAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<CacheEntry<Object, Object>> getEntries(
+        @Nullable Collection keys) throws IgniteCheckedException {
+        return delegate.getEntries(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map<Object, Object>> getAllAsync(@Nullable Collection keys) {
+        return delegate.getAllAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Collection<CacheEntry<Object,Object>>> getEntriesAsync(
+        @Nullable Collection keys
+    ) {
+        return delegate.getEntriesAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndPut(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndPut(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndPutAsync(Object key, Object val) {
+        return delegate.getAndPutAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean put(Object key, Object val) throws IgniteCheckedException {
+        return delegate.put(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> putAsync(Object key, Object val) {
+        return delegate.putAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndPutIfAbsent(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndPutIfAbsent(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndPutIfAbsentAsync(Object key, Object val) {
+        return delegate.getAndPutIfAbsentAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean putIfAbsent(Object key, Object val) throws IgniteCheckedException {
+        return delegate.putIfAbsent(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(Object key, Object val) {
+        return delegate.putIfAbsentAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndReplace(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndReplace(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndReplaceAsync(Object key, Object val) {
+        return delegate.getAndReplaceAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean replace(Object key, Object val) throws IgniteCheckedException {
+        return delegate.replace(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> replaceAsync(Object key, Object val) {
+        return delegate.replaceAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean replace(Object key, Object oldVal, Object newVal) throws IgniteCheckedException {
+        return delegate.replace(keyTransformer.transform(key), oldVal, newVal);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> replaceAsync(Object key, Object oldVal, Object newVal) {
+        return delegate.replaceAsync(keyTransformer.transform(key), oldVal, newVal);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void putAll(@Nullable Map m) throws IgniteCheckedException {
+        delegate.putAll(transform(m));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> putAllAsync(@Nullable Map m) {
+        return delegate.putAllAsync(transform(m));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set keySet() {
+        return delegate.keySet();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set keySetx() {
+        return delegate.keySetx();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set primaryKeySet() {
+        return delegate.primaryKeySet();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable values() {
+        return delegate.values();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set<Cache.Entry<Object, Object>> entrySet() {
+        return delegate.entrySet();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Set<Cache.Entry<Object,Object>> entrySet(int part) {
+        return delegate.entrySet(part);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set<Cache.Entry<Object, Object>> entrySetx(CacheEntryPredicate... filter) {
+        return delegate.entrySetx(filter);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Transaction txStart(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation
+    ) {
+        return delegate.txStart(concurrency, isolation);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalTx txStartEx(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation
+    ) {
+        return delegate.txStartEx(concurrency, isolation);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Transaction txStart(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation,
+        long timeout,
+        int txSize
+    ) {
+        return delegate.txStart(concurrency, isolation, timeout, txSize);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Transaction tx() {
+        return delegate.tx();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean evict(Object key) {
+        return delegate.evict(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void evictAll(@Nullable Collection keys) {
+        delegate.evictAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearLocally(boolean srv, boolean near, boolean readers) {
+        delegate.clearLocally(srv, near, readers);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean clearLocally(Object key) {
+        return delegate.clearLocally(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearLocallyAll(Set keys, boolean srv, boolean near, boolean readers) {
+        delegate.clearLocallyAll((Set<?>)transform(keys), srv, near, readers);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear(Object key) throws IgniteCheckedException {
+        delegate.clear(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearAll(Set keys) throws IgniteCheckedException {
+        delegate.clearAll((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear() throws IgniteCheckedException {
+        delegate.clear();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAsync() {
+        return delegate.clearAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAsync(Object key) {
+        return delegate.clearAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAllAsync(Set keys) {
+        return delegate.clearAllAsync((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndRemove(Object key) throws IgniteCheckedException {
+        return delegate.getAndRemove(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndRemoveAsync(Object key) {
+        return delegate.getAndRemoveAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(Object key) throws IgniteCheckedException {
+        return delegate.remove(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> removeAsync(Object key) {
+        return delegate.removeAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(Object key, Object val) throws IgniteCheckedException {
+        return delegate.remove(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> removeAsync(Object key, Object val) {
+        return delegate.removeAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.removeAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllAsync(@Nullable Collection keys) {
+        return delegate.removeAllAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAll() throws IgniteCheckedException {
+        delegate.removeAll();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllAsync() {
+        return delegate.removeAllAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lock(Object key, long timeout) throws IgniteCheckedException {
+        return delegate.lock(keyTransformer.transform(key), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> lockAsync(Object key, long timeout) {
+        return delegate.lockAsync(keyTransformer.transform(key), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lockAll(@Nullable Collection keys, long timeout) throws IgniteCheckedException {
+        return delegate.lockAll(transform(keys), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> lockAllAsync(@Nullable Collection keys, long timeout) {
+        return delegate.lockAllAsync(transform(keys), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlock(Object key) throws IgniteCheckedException {
+        delegate.unlock(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlockAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.unlockAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLocked(Object key) {
+        return delegate.isLocked(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLockedByThread(Object key) {
+        return delegate.isLockedByThread(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size() {
+        return delegate.size();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong() {
+        return delegate.sizeLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int localSize(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSize(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long localSizeLong(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSizeLong(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long localSizeLong(int partition, CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSizeLong(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.size(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.sizeLong(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong(int partition, CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.sizeLong(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Integer> sizeAsync(CachePeekMode[] peekModes) {
+        return delegate.sizeAsync(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Long> sizeLongAsync(CachePeekMode[] peekModes) {
+        return delegate.sizeLongAsync(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Long> sizeLongAsync(int partition, CachePeekMode[] peekModes) {
+        return delegate.sizeLongAsync(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int nearSize() {
+        return delegate.nearSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int primarySize() {
+        return delegate.primarySize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long primarySizeLong() {
+        return delegate.primarySizeLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheConfiguration configuration() {
+        return delegate.configuration();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Affinity affinity() {
+        return delegate.affinity();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics clusterMetrics() {
+        return delegate.clusterMetrics();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics clusterMetrics(ClusterGroup grp) {
+        return delegate.clusterMetrics(grp);
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics localMetrics() {
+        return delegate.localMetrics();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetricsMXBean clusterMxBean() {
+        return delegate.clusterMxBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetricsMXBean localMxBean() {
+        return delegate.localMxBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long offHeapEntriesCount() {
+        return delegate.offHeapEntriesCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long offHeapAllocatedSize() {
+        return delegate.offHeapAllocatedSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> rebalance() {
+        return delegate.rebalance();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache forSubjectId(UUID subjId) {
+        return delegate.forSubjectId(subjId);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getForcePrimary(Object key) throws IgniteCheckedException {
+        return delegate.getForcePrimary(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getForcePrimaryAsync(Object key) {
+        return delegate.getForcePrimaryAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map getAllOutTx(Set keys) throws IgniteCheckedException {
+        return delegate.getAllOutTx((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map<Object, Object>> getAllOutTxAsync(Set keys) {
+        return delegate.getAllOutTxAsync((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isIgfsDataCache() {
+        return delegate.isIgfsDataCache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long igfsDataSpaceUsed() {
+        return delegate.igfsDataSpaceUsed();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long igfsDataSpaceMax() {
+        return delegate.igfsDataSpaceMax();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMongoDataCache() {
+        return delegate.isMongoDataCache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMongoMetaCache() {
+        return delegate.isMongoMetaCache();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public ExpiryPolicy expiry() {
+        return delegate.expiry();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache withExpiryPolicy(ExpiryPolicy plc) {
+        return delegate.withExpiryPolicy(plc);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache withNoRetries() {
+        return delegate.withNoRetries();
+    }
+
+    /** {@inheritDoc} */
+    @Override public GridCacheContext context() {
+        return delegate.context();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void localLoadCache(
+        @Nullable IgniteBiPredicate p,
+        @Nullable Object... args
+    ) throws IgniteCheckedException {
+        delegate.localLoadCache(p, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> localLoadCacheAsync(
+        @Nullable IgniteBiPredicate p,
+        @Nullable Object... args
+    ) {
+        return delegate.localLoadCacheAsync(p, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Object getTopologySafe(Object key) throws IgniteCheckedException {
+        return delegate.getTopologySafe(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object tryGetAndPut(Object key, Object val) throws IgniteCheckedException {
+        return delegate.tryGetAndPut(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryProcessorResult invoke(
+        @Nullable AffinityTopologyVersion topVer,
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) throws IgniteCheckedException {
+        return delegate.invoke(topVer, key, entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map> invokeAllAsync(Map map, Object... args) {
+        return delegate.invokeAllAsync(map, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map invokeAll(Map map, Object... args) throws IgniteCheckedException {
+        return delegate.invokeAll(map, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map> invokeAllAsync(Set keys, EntryProcessor entryProcessor, Object... args) {
+        return delegate.invokeAllAsync((Set<?>)transform(keys), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map invokeAll(Set keys, EntryProcessor entryProcessor, Object... args) throws IgniteCheckedException {
+        return delegate.invokeAll((Set<?>)transform(keys), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<EntryProcessorResult> invokeAsync(
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) {
+        return delegate.invokeAsync(keyTransformer.transform(key), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryProcessorResult invoke(
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) throws IgniteCheckedException {
+        return delegate.invoke(keyTransformer.transform(key), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllConflictAsync(Map drMap) throws IgniteCheckedException {
+        return delegate.removeAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAllConflict(Map drMap) throws IgniteCheckedException {
+        delegate.removeAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> putAllConflictAsync(Map drMap) throws IgniteCheckedException {
+        return delegate.putAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void putAllConflict(Map drMap) throws IgniteCheckedException {
+        delegate.putAllConflict(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache keepBinary() {
+        return delegate.keepBinary();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache cache() {
+        return delegate.cache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterator iterator() {
+        return delegate.iterator();
+    }
+
+    /**
+     * @param keys Keys.
+     */
+    private Collection<Object> transform(Collection<Object> keys) {
+        Collection<Object> res = new LinkedList<>();
+
+        for (Object o : keys)
+            res.add(keyTransformer.transform(o));
+
+        return res;
+    }
+
+    /**
+     * @param map Map.
+     */
+    private Map<Object, Object> transform(Map<Object, Object> map) {
+        Map<Object, Object> res = new HashMap<>();
+
+        Set<Map.Entry<Object, Object>> ents = map.entrySet();
+
+        for (Map.Entry<Object, Object> e : ents)
+            res.put(keyTransformer.transform(e.getKey()), e.getValue());
+
+        return res;
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
index 045f401d92f80..eb35a2c4709d7 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.CollectionRegion;
@@ -72,7 +71,7 @@ public class HibernateCollectionRegion extends HibernateTransactionalDataRegion
      * @param dataDesc Region data description.
      */
     public HibernateCollectionRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
index 1ceda144a8355..ad5b1919f51b3 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.EntityRegion;
@@ -62,7 +61,7 @@ public class HibernateEntityRegion extends HibernateTransactionalDataRegion impl
      * @param dataDesc Region data description.
      */
     public HibernateEntityRegion(HibernateRegionFactory factory, String name, Ignite ignite,
-        IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
index fbac624f66af1..2f1a11dc8baa3 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.GeneralDataRegion;
 import org.hibernate.cache.spi.QueryResultsRegion;
@@ -38,7 +37,7 @@ public class HibernateGeneralDataRegion extends HibernateRegion implements Gener
      * @param cache Region cache.
      */
     public HibernateGeneralDataRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite, HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
new file mode 100644
index 0000000000000..97fc0e9eb15e6
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+/**
+ * An interface for transforming hibernate keys to Ignite keys.
+ */
+public interface HibernateKeyTransformer {
+    /**
+     * @param key Hibernate key.
+     * @return Transformed key.
+     */
+    public Object transform(Object key);
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
new file mode 100644
index 0000000000000..7de440ebb032c
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Hibernate cache key wrapper.
+ */
+public class HibernateKeyWrapper {
+    /** Key. */
+    private final Object key;
+
+    /** Entry. */
+    private final String entry;
+
+    /** */
+    private final String tenantId;
+
+    /**
+     * @param key Key.
+     * @param entry Entry.
+     * @param tenantId Tenant ID.
+     */
+    HibernateKeyWrapper(Object key, String entry, String tenantId) {
+        this.key = key;
+        this.entry = entry;
+        this.tenantId = tenantId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o) return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        HibernateKeyWrapper that = (HibernateKeyWrapper) o;
+
+        return (key != null ? key.equals(that.key) : that.key == null) &&
+            (entry != null ? entry.equals(that.entry) : that.entry == null) &&
+            (tenantId != null ? tenantId.equals(that.tenantId) : that.tenantId == null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = key != null ? key.hashCode() : 0;
+        res = 31 * res + (entry != null ? entry.hashCode() : 0);
+        res = 31 * res + (tenantId != null ? tenantId.hashCode() : 0);
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HibernateKeyWrapper.class, this);
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
index 99d5348dea479..862a4228f2767 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.NaturalIdRegion;
@@ -52,7 +51,7 @@ public class HibernateNaturalIdRegion extends HibernateTransactionalDataRegion i
      * @param dataDesc Region data description.
      */
     public HibernateNaturalIdRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
index 1cb8d48e0be2f..a36d7e786d739 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
@@ -21,7 +21,6 @@
 import java.util.Set;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridLeanMap;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.typedef.F;
@@ -67,7 +66,7 @@ public class HibernateNonStrictAccessStrategy extends HibernateAccessStrategyAda
      * @param cache Cache.
      * @param writeCtx Thread local instance used to track updates done during one Hibernate transaction.
      */
-    protected HibernateNonStrictAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache, ThreadLocal writeCtx) {
+    protected HibernateNonStrictAccessStrategy(Ignite ignite, HibernateCacheProxy cache, ThreadLocal writeCtx) {
         super(ignite, cache);
 
         this.writeCtx = (ThreadLocal<WriteContext>)writeCtx;
@@ -212,7 +211,7 @@ void removed(Object key) {
          * @param cache Cache.
          * @throws IgniteCheckedException If failed.
          */
-        void updateCache(IgniteInternalCache<Object, Object> cache) throws IgniteCheckedException {
+        void updateCache(HibernateCacheProxy cache) throws IgniteCheckedException {
             if (!F.isEmpty(rmvs))
                 cache.removeAll(rmvs);
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
index e3303a71ff413..0b9a43d6a13d6 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.Query;
 import org.hibernate.cache.spi.QueryResultsRegion;
 
@@ -65,7 +64,7 @@ public class HibernateQueryResultsRegion extends HibernateGeneralDataRegion impl
      * @param cache Region cache.
      */
     public HibernateQueryResultsRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite, HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
index 58a2c4b8779d4..cdef80e0a0c0b 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.access.AccessType;
 import org.hibernate.cache.spi.access.SoftLock;
@@ -60,7 +59,7 @@ public class HibernateReadOnlyAccessStrategy extends HibernateAccessStrategyAdap
      * @param ignite Grid.
      * @param cache Cache.
      */
-    public HibernateReadOnlyAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    public HibernateReadOnlyAccessStrategy(Ignite ignite, HibernateCacheProxy cache) {
         super(ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
index bbb1d4e5dfdc2..625b05061ff0f 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
@@ -21,7 +21,6 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.transactions.Transaction;
 import org.hibernate.cache.CacheException;
@@ -68,7 +67,7 @@ public class HibernateReadWriteAccessStrategy extends HibernateAccessStrategyAda
      * @param cache Cache.
      * @param txCtx Thread local instance used to track updates done during one Hibernate transaction.
      */
-    protected HibernateReadWriteAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache, ThreadLocal txCtx) {
+    protected HibernateReadWriteAccessStrategy(Ignite ignite, HibernateCacheProxy cache, ThreadLocal txCtx) {
         super(ignite, cache);
 
         this.txCtx = (ThreadLocal<TxContext>)txCtx;
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
index 27479e90d8b7b..11a96d09096fb 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
@@ -20,7 +20,6 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.Region;
 
@@ -35,7 +34,7 @@ public class HibernateRegion implements Region {
     private final String name;
 
     /** Cache instance. */
-    protected final IgniteInternalCache<Object, Object> cache;
+    protected final HibernateCacheProxy cache;
 
     /** Grid instance. */
     protected Ignite ignite;
@@ -46,8 +45,7 @@ public class HibernateRegion implements Region {
      * @param ignite Grid.
      * @param cache Region cache.
      */
-    public HibernateRegion(HibernateRegionFactory factory, String name, Ignite ignite,
-        IgniteInternalCache<Object, Object> cache) {
+    public HibernateRegion(HibernateRegionFactory factory, String name, Ignite ignite, HibernateCacheProxy cache) {
         this.factory = factory;
         this.name = name;
         this.ignite = ignite;
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
index 825abee7e2590..4e4be36ec7c49 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.util.typedef.G;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
+import org.hibernate.cache.spi.CacheKey;
 import org.hibernate.cache.spi.CollectionRegion;
 import org.hibernate.cache.spi.EntityRegion;
 import org.hibernate.cache.spi.NaturalIdRegion;
@@ -88,7 +89,7 @@ public class HibernateRegionFactory implements RegionFactory {
     private Ignite ignite;
 
     /** Default cache. */
-    private IgniteInternalCache<Object, Object> dfltCache;
+    private HibernateCacheProxy dfltCache;
 
     /** Default region access type. */
     private AccessType dfltAccessType;
@@ -99,6 +100,23 @@ public class HibernateRegionFactory implements RegionFactory {
     /** Map needed to provide the same transaction context for different regions. */
     private final ThreadLocal threadLoc = new ThreadLocal();
 
+    /** Key transformer. */
+    private final HibernateKeyTransformer hibernate4transformer = new HibernateKeyTransformer() {
+        @Override public Object transform(Object key) {
+            if (key instanceof CacheKey) {
+                CacheKey cacheKey = (CacheKey)key;
+
+                return new HibernateKeyWrapper(
+                    cacheKey.getKey(),
+                    cacheKey.getEntityOrRoleName(),
+                    cacheKey.getTenantId()
+                );
+            }
+
+            return key;
+        }
+    };
+
     /** {@inheritDoc} */
     @Override public void start(Settings settings, Properties props) throws CacheException {
         String gridCfg = props.getProperty(GRID_CONFIG_PROPERTY);
@@ -138,10 +156,12 @@ public class HibernateRegionFactory implements RegionFactory {
         String dfltCacheName = props.getProperty(DFLT_CACHE_NAME_PROPERTY);
 
         if (dfltCacheName != null) {
-            dfltCache = ((IgniteKernal)ignite).getCache(dfltCacheName);
+            IgniteInternalCache<Object, Object> dfltCache = ((IgniteKernal)ignite).getCache(dfltCacheName);
 
             if (dfltCache == null)
                 throw new CacheException("Cache specified as default is not configured: " + dfltCacheName);
+
+            this.dfltCache = new HibernateCacheProxy(dfltCache, hibernate4transformer);
         }
 
         IgniteLogger log = ignite.log().getLogger(HibernateRegionFactory.class);
@@ -152,6 +172,7 @@ public class HibernateRegionFactory implements RegionFactory {
 
     /** {@inheritDoc} */
     @Override public void stop() {
+        // No-op.
     }
 
     /** {@inheritDoc} */
@@ -213,7 +234,7 @@ ThreadLocal threadLocalForCache(String cacheName) {
      * @return Cache for given region.
      * @throws CacheException If cache for given region is not configured.
      */
-    private IgniteInternalCache<Object, Object> regionCache(String regionName) throws CacheException {
+    private HibernateCacheProxy regionCache(String regionName) throws CacheException {
         String cacheName = regionCaches.get(regionName);
 
         if (cacheName == null) {
@@ -228,6 +249,6 @@ private IgniteInternalCache<Object, Object> regionCache(String regionName) throw
         if (cache == null)
             throw new CacheException("Cache '" + cacheName + "' for region '" + regionName + "' is not configured.");
 
-        return cache;
+        return new HibernateCacheProxy(cache, hibernate4transformer);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
index 4cedae2070e31..8b4c243277ec9 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.spi.TimestampsRegion;
 
 /**
@@ -34,7 +33,7 @@ public class HibernateTimestampsRegion extends HibernateGeneralDataRegion implem
      * @param cache Region cache.
      */
     public HibernateTimestampsRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite,  HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
index 80f75a71afed6..ca5284917aa68 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
@@ -61,7 +61,7 @@ public class HibernateTransactionalAccessStrategy extends HibernateAccessStrateg
      * @param ignite Grid.
      * @param cache Cache.
      */
-    public HibernateTransactionalAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    public HibernateTransactionalAccessStrategy(Ignite ignite, HibernateCacheProxy cache) {
         super(ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
index ed2ee01cf2280..581076a3fdbb6 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.TransactionConfiguration;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.CollectionRegion;
@@ -48,7 +47,7 @@ public class HibernateTransactionalDataRegion extends HibernateRegion implements
      * @param dataDesc Region data description.
      */
     public HibernateTransactionalDataRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache);
 
         this.dataDesc = dataDesc;
diff --git a/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java b/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java
new file mode 100644
index 0000000000000..3791baed9c93c
--- /dev/null
+++ b/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.testframework.config.GridTestProperties;
+
+/**
+ *
+ */
+public class IgniteBinaryHibernateTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception If failed.
+     */
+    public static TestSuite suite() throws Exception {
+        GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, BinaryMarshaller.class.getName());
+
+        return IgniteHibernateTestSuite.suite();
+    }
+}

From 16067300c9124b79bfee42139eb881ae585c0914 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dgovorukhin@gridgain.com>
Date: Fri, 7 Apr 2017 14:28:22 +0300
Subject: [PATCH 288/516] IGNITE-4889 - Changed Hibernate integration to use
 custom keys

(cherry picked from commit 6b62a20)
---
 .../HibernateAccessStrategyAdapter.java       |   8 +-
 .../cache/hibernate/HibernateCacheProxy.java  | 818 ++++++++++++++++++
 .../hibernate/HibernateCollectionRegion.java  |   3 +-
 .../hibernate/HibernateEntityRegion.java      |   3 +-
 .../hibernate/HibernateGeneralDataRegion.java |   3 +-
 .../hibernate/HibernateKeyTransformer.java    |  29 +
 .../cache/hibernate/HibernateKeyWrapper.java  |  72 ++
 .../hibernate/HibernateNaturalIdRegion.java   |   3 +-
 .../HibernateNonStrictAccessStrategy.java     |   5 +-
 .../HibernateQueryResultsRegion.java          |   3 +-
 .../HibernateReadOnlyAccessStrategy.java      |   3 +-
 .../HibernateReadWriteAccessStrategy.java     |   3 +-
 .../cache/hibernate/HibernateRegion.java      |   6 +-
 .../hibernate/HibernateRegionFactory.java     |  29 +-
 .../hibernate/HibernateTimestampsRegion.java  |   3 +-
 .../HibernateTransactionalAccessStrategy.java |   2 +-
 .../HibernateTransactionalDataRegion.java     |   3 +-
 .../IgniteBinaryHibernateTestSuite.java       |  37 +
 18 files changed, 1000 insertions(+), 33 deletions(-)
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
 create mode 100644 modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java

diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
index 27734d9ad9ca5..f6c1d0e15eb43 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
@@ -92,7 +92,7 @@
  */
 public abstract class HibernateAccessStrategyAdapter {
     /** */
-    protected final IgniteInternalCache<Object, Object> cache;
+    protected final HibernateCacheProxy cache;
 
     /** Grid. */
     protected final Ignite ignite;
@@ -104,7 +104,7 @@ public abstract class HibernateAccessStrategyAdapter {
      * @param ignite Grid.
      * @param cache Cache.
      */
-    protected HibernateAccessStrategyAdapter(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    protected HibernateAccessStrategyAdapter(Ignite ignite, HibernateCacheProxy cache) {
         this.cache = cache;
         this.ignite = ignite;
 
@@ -292,8 +292,10 @@ protected final void removeAll() throws CacheException {
      * @param key Key.
      * @throws CacheException If failed.
      */
-    static void evict(Ignite ignite, IgniteInternalCache<Object,Object> cache, Object key) throws CacheException {
+    static void evict(Ignite ignite, HibernateCacheProxy cache, Object key) throws CacheException {
         try {
+            key = cache.keyTransformer().transform(key);
+
             ignite.compute(ignite.cluster()).call(new ClearKeyCallable(key, cache.name()));
         }
         catch (IgniteException e) {
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
new file mode 100644
index 0000000000000..871c4a100ac6f
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
@@ -0,0 +1,818 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorResult;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheEntry;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.mxbean.CacheMetricsMXBean;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Hibernate cache proxy.
+ */
+public class HibernateCacheProxy implements IgniteInternalCache<Object, Object> {
+    /** Delegate. */
+    private final IgniteInternalCache<Object, Object> delegate;
+
+    /** Transformer. */
+    private final HibernateKeyTransformer keyTransformer;
+
+    /**
+     * @param delegate Delegate.
+     * @param keyTransformer Key keyTransformer.
+     */
+    HibernateCacheProxy(
+        IgniteInternalCache<Object, Object> delegate,
+        HibernateKeyTransformer keyTransformer
+    ) {
+        assert delegate != null;
+        assert keyTransformer != null;
+
+        this.delegate = delegate;
+        this.keyTransformer = keyTransformer;
+    }
+
+    /**
+     * @return HibernateKeyTransformer
+     */
+    HibernateKeyTransformer keyTransformer(){
+        return keyTransformer;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String name() {
+        return delegate.name();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean skipStore() {
+        return delegate.skipStore();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache setSkipStore(boolean skipStore) {
+        return delegate.setSkipStore(skipStore);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean containsKey(Object key) {
+        return delegate.containsKey(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> containsKeyAsync(Object key) {
+        return delegate.containsKeyAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void promoteAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.promoteAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public long overflowSize() throws IgniteCheckedException {
+        return delegate.overflowSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long swapSize() throws IgniteCheckedException {
+        return delegate.swapSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long swapKeys() throws IgniteCheckedException {
+        return delegate.swapKeys();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean containsKeys(Collection keys) {
+        return delegate.containsKey(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> containsKeysAsync(Collection keys) {
+        return delegate.containsKeysAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object localPeek(
+        Object key,
+        CachePeekMode[] peekModes,
+        @Nullable IgniteCacheExpiryPolicy plc
+    ) throws IgniteCheckedException {
+        return delegate.localPeek(keyTransformer.transform(key), peekModes, plc);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable<Cache.Entry<Object, Object>> localEntries(
+        CachePeekMode[] peekModes
+    ) throws IgniteCheckedException {
+        return delegate.localEntries(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object get(Object key) throws IgniteCheckedException {
+        return delegate.get(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public CacheEntry getEntry(Object key) throws IgniteCheckedException {
+        return delegate.getEntry(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAsync(Object key) {
+        return delegate.getAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<CacheEntry<Object, Object>> getEntryAsync(Object key) {
+        return delegate.getEntryAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map getAll(@Nullable Collection keys) throws IgniteCheckedException {
+        return delegate.getAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<CacheEntry<Object, Object>> getEntries(
+        @Nullable Collection keys) throws IgniteCheckedException {
+        return delegate.getEntries(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map<Object, Object>> getAllAsync(@Nullable Collection keys) {
+        return delegate.getAllAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Collection<CacheEntry<Object,Object>>> getEntriesAsync(
+        @Nullable Collection keys
+    ) {
+        return delegate.getEntriesAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndPut(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndPut(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndPutAsync(Object key, Object val) {
+        return delegate.getAndPutAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean put(Object key, Object val) throws IgniteCheckedException {
+        return delegate.put(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> putAsync(Object key, Object val) {
+        return delegate.putAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndPutIfAbsent(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndPutIfAbsent(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndPutIfAbsentAsync(Object key, Object val) {
+        return delegate.getAndPutIfAbsentAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean putIfAbsent(Object key, Object val) throws IgniteCheckedException {
+        return delegate.putIfAbsent(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(Object key, Object val) {
+        return delegate.putIfAbsentAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndReplace(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndReplace(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndReplaceAsync(Object key, Object val) {
+        return delegate.getAndReplaceAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean replace(Object key, Object val) throws IgniteCheckedException {
+        return delegate.replace(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> replaceAsync(Object key, Object val) {
+        return delegate.replaceAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean replace(Object key, Object oldVal, Object newVal) throws IgniteCheckedException {
+        return delegate.replace(keyTransformer.transform(key), oldVal, newVal);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> replaceAsync(Object key, Object oldVal, Object newVal) {
+        return delegate.replaceAsync(keyTransformer.transform(key), oldVal, newVal);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void putAll(@Nullable Map m) throws IgniteCheckedException {
+        delegate.putAll(transform(m));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> putAllAsync(@Nullable Map m) {
+        return delegate.putAllAsync(transform(m));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set keySet() {
+        return delegate.keySet();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set keySetx() {
+        return delegate.keySetx();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set primaryKeySet() {
+        return delegate.primaryKeySet();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable values() {
+        return delegate.values();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set<Cache.Entry<Object, Object>> entrySet() {
+        return delegate.entrySet();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Set<Cache.Entry<Object,Object>> entrySet(int part) {
+        return delegate.entrySet(part);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set<Cache.Entry<Object, Object>> entrySetx(CacheEntryPredicate... filter) {
+        return delegate.entrySetx(filter);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Transaction txStart(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation
+    ) {
+        return delegate.txStart(concurrency, isolation);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalTx txStartEx(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation
+    ) {
+        return delegate.txStartEx(concurrency, isolation);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Transaction txStart(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation,
+        long timeout,
+        int txSize
+    ) {
+        return delegate.txStart(concurrency, isolation, timeout, txSize);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Transaction tx() {
+        return delegate.tx();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean evict(Object key) {
+        return delegate.evict(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void evictAll(@Nullable Collection keys) {
+        delegate.evictAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearLocally(boolean srv, boolean near, boolean readers) {
+        delegate.clearLocally(srv, near, readers);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean clearLocally(Object key) {
+        return delegate.clearLocally(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearLocallyAll(Set keys, boolean srv, boolean near, boolean readers) {
+        delegate.clearLocallyAll((Set<?>)transform(keys), srv, near, readers);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear(Object key) throws IgniteCheckedException {
+        delegate.clear(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearAll(Set keys) throws IgniteCheckedException {
+        delegate.clearAll((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear() throws IgniteCheckedException {
+        delegate.clear();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAsync() {
+        return delegate.clearAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAsync(Object key) {
+        return delegate.clearAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAllAsync(Set keys) {
+        return delegate.clearAllAsync((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndRemove(Object key) throws IgniteCheckedException {
+        return delegate.getAndRemove(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndRemoveAsync(Object key) {
+        return delegate.getAndRemoveAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(Object key) throws IgniteCheckedException {
+        return delegate.remove(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> removeAsync(Object key) {
+        return delegate.removeAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(Object key, Object val) throws IgniteCheckedException {
+        return delegate.remove(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> removeAsync(Object key, Object val) {
+        return delegate.removeAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.removeAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllAsync(@Nullable Collection keys) {
+        return delegate.removeAllAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAll() throws IgniteCheckedException {
+        delegate.removeAll();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllAsync() {
+        return delegate.removeAllAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lock(Object key, long timeout) throws IgniteCheckedException {
+        return delegate.lock(keyTransformer.transform(key), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> lockAsync(Object key, long timeout) {
+        return delegate.lockAsync(keyTransformer.transform(key), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lockAll(@Nullable Collection keys, long timeout) throws IgniteCheckedException {
+        return delegate.lockAll(transform(keys), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> lockAllAsync(@Nullable Collection keys, long timeout) {
+        return delegate.lockAllAsync(transform(keys), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlock(Object key) throws IgniteCheckedException {
+        delegate.unlock(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlockAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.unlockAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLocked(Object key) {
+        return delegate.isLocked(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLockedByThread(Object key) {
+        return delegate.isLockedByThread(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size() {
+        return delegate.size();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong() {
+        return delegate.sizeLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int localSize(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSize(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long localSizeLong(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSizeLong(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long localSizeLong(int partition, CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSizeLong(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.size(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.sizeLong(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong(int partition, CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.sizeLong(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Integer> sizeAsync(CachePeekMode[] peekModes) {
+        return delegate.sizeAsync(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Long> sizeLongAsync(CachePeekMode[] peekModes) {
+        return delegate.sizeLongAsync(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Long> sizeLongAsync(int partition, CachePeekMode[] peekModes) {
+        return delegate.sizeLongAsync(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int nearSize() {
+        return delegate.nearSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int primarySize() {
+        return delegate.primarySize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long primarySizeLong() {
+        return delegate.primarySizeLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheConfiguration configuration() {
+        return delegate.configuration();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Affinity affinity() {
+        return delegate.affinity();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics clusterMetrics() {
+        return delegate.clusterMetrics();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics clusterMetrics(ClusterGroup grp) {
+        return delegate.clusterMetrics(grp);
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics localMetrics() {
+        return delegate.localMetrics();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetricsMXBean clusterMxBean() {
+        return delegate.clusterMxBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetricsMXBean localMxBean() {
+        return delegate.localMxBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long offHeapEntriesCount() {
+        return delegate.offHeapEntriesCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long offHeapAllocatedSize() {
+        return delegate.offHeapAllocatedSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> rebalance() {
+        return delegate.rebalance();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache forSubjectId(UUID subjId) {
+        return delegate.forSubjectId(subjId);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getForcePrimary(Object key) throws IgniteCheckedException {
+        return delegate.getForcePrimary(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getForcePrimaryAsync(Object key) {
+        return delegate.getForcePrimaryAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map getAllOutTx(Set keys) throws IgniteCheckedException {
+        return delegate.getAllOutTx((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map<Object, Object>> getAllOutTxAsync(Set keys) {
+        return delegate.getAllOutTxAsync((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isIgfsDataCache() {
+        return delegate.isIgfsDataCache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long igfsDataSpaceUsed() {
+        return delegate.igfsDataSpaceUsed();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long igfsDataSpaceMax() {
+        return delegate.igfsDataSpaceMax();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMongoDataCache() {
+        return delegate.isMongoDataCache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMongoMetaCache() {
+        return delegate.isMongoMetaCache();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public ExpiryPolicy expiry() {
+        return delegate.expiry();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache withExpiryPolicy(ExpiryPolicy plc) {
+        return delegate.withExpiryPolicy(plc);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache withNoRetries() {
+        return delegate.withNoRetries();
+    }
+
+    /** {@inheritDoc} */
+    @Override public GridCacheContext context() {
+        return delegate.context();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void localLoadCache(
+        @Nullable IgniteBiPredicate p,
+        @Nullable Object... args
+    ) throws IgniteCheckedException {
+        delegate.localLoadCache(p, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> localLoadCacheAsync(
+        @Nullable IgniteBiPredicate p,
+        @Nullable Object... args
+    ) {
+        return delegate.localLoadCacheAsync(p, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Object getTopologySafe(Object key) throws IgniteCheckedException {
+        return delegate.getTopologySafe(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object tryGetAndPut(Object key, Object val) throws IgniteCheckedException {
+        return delegate.tryGetAndPut(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryProcessorResult invoke(
+        @Nullable AffinityTopologyVersion topVer,
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) throws IgniteCheckedException {
+        return delegate.invoke(topVer, key, entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map> invokeAllAsync(Map map, Object... args) {
+        return delegate.invokeAllAsync(map, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map invokeAll(Map map, Object... args) throws IgniteCheckedException {
+        return delegate.invokeAll(map, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map> invokeAllAsync(Set keys, EntryProcessor entryProcessor, Object... args) {
+        return delegate.invokeAllAsync((Set<?>)transform(keys), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map invokeAll(Set keys, EntryProcessor entryProcessor, Object... args) throws IgniteCheckedException {
+        return delegate.invokeAll((Set<?>)transform(keys), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<EntryProcessorResult> invokeAsync(
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) {
+        return delegate.invokeAsync(keyTransformer.transform(key), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryProcessorResult invoke(
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) throws IgniteCheckedException {
+        return delegate.invoke(keyTransformer.transform(key), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllConflictAsync(Map drMap) throws IgniteCheckedException {
+        return delegate.removeAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAllConflict(Map drMap) throws IgniteCheckedException {
+        delegate.removeAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> putAllConflictAsync(Map drMap) throws IgniteCheckedException {
+        return delegate.putAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void putAllConflict(Map drMap) throws IgniteCheckedException {
+        delegate.putAllConflict(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache keepBinary() {
+        return delegate.keepBinary();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache cache() {
+        return delegate.cache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterator iterator() {
+        return delegate.iterator();
+    }
+
+    /**
+     * @param keys Keys.
+     */
+    private Collection<Object> transform(Collection<Object> keys) {
+        Collection<Object> res = new LinkedList<>();
+
+        for (Object o : keys)
+            res.add(keyTransformer.transform(o));
+
+        return res;
+    }
+
+    /**
+     * @param map Map.
+     */
+    private Map<Object, Object> transform(Map<Object, Object> map) {
+        Map<Object, Object> res = new HashMap<>();
+
+        Set<Map.Entry<Object, Object>> ents = map.entrySet();
+
+        for (Map.Entry<Object, Object> e : ents)
+            res.put(keyTransformer.transform(e.getKey()), e.getValue());
+
+        return res;
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
index 045f401d92f80..eb35a2c4709d7 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.CollectionRegion;
@@ -72,7 +71,7 @@ public class HibernateCollectionRegion extends HibernateTransactionalDataRegion
      * @param dataDesc Region data description.
      */
     public HibernateCollectionRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
index 1ceda144a8355..ad5b1919f51b3 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.EntityRegion;
@@ -62,7 +61,7 @@ public class HibernateEntityRegion extends HibernateTransactionalDataRegion impl
      * @param dataDesc Region data description.
      */
     public HibernateEntityRegion(HibernateRegionFactory factory, String name, Ignite ignite,
-        IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
index fbac624f66af1..2f1a11dc8baa3 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.GeneralDataRegion;
 import org.hibernate.cache.spi.QueryResultsRegion;
@@ -38,7 +37,7 @@ public class HibernateGeneralDataRegion extends HibernateRegion implements Gener
      * @param cache Region cache.
      */
     public HibernateGeneralDataRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite, HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
new file mode 100644
index 0000000000000..97fc0e9eb15e6
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+/**
+ * An interface for transforming hibernate keys to Ignite keys.
+ */
+public interface HibernateKeyTransformer {
+    /**
+     * @param key Hibernate key.
+     * @return Transformed key.
+     */
+    public Object transform(Object key);
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
new file mode 100644
index 0000000000000..7de440ebb032c
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Hibernate cache key wrapper.
+ */
+public class HibernateKeyWrapper {
+    /** Key. */
+    private final Object key;
+
+    /** Entry. */
+    private final String entry;
+
+    /** */
+    private final String tenantId;
+
+    /**
+     * @param key Key.
+     * @param entry Entry.
+     * @param tenantId Tenant ID.
+     */
+    HibernateKeyWrapper(Object key, String entry, String tenantId) {
+        this.key = key;
+        this.entry = entry;
+        this.tenantId = tenantId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o) return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        HibernateKeyWrapper that = (HibernateKeyWrapper) o;
+
+        return (key != null ? key.equals(that.key) : that.key == null) &&
+            (entry != null ? entry.equals(that.entry) : that.entry == null) &&
+            (tenantId != null ? tenantId.equals(that.tenantId) : that.tenantId == null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = key != null ? key.hashCode() : 0;
+        res = 31 * res + (entry != null ? entry.hashCode() : 0);
+        res = 31 * res + (tenantId != null ? tenantId.hashCode() : 0);
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HibernateKeyWrapper.class, this);
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
index 99d5348dea479..862a4228f2767 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.NaturalIdRegion;
@@ -52,7 +51,7 @@ public class HibernateNaturalIdRegion extends HibernateTransactionalDataRegion i
      * @param dataDesc Region data description.
      */
     public HibernateNaturalIdRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
index 1cb8d48e0be2f..a36d7e786d739 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
@@ -21,7 +21,6 @@
 import java.util.Set;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridLeanMap;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.typedef.F;
@@ -67,7 +66,7 @@ public class HibernateNonStrictAccessStrategy extends HibernateAccessStrategyAda
      * @param cache Cache.
      * @param writeCtx Thread local instance used to track updates done during one Hibernate transaction.
      */
-    protected HibernateNonStrictAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache, ThreadLocal writeCtx) {
+    protected HibernateNonStrictAccessStrategy(Ignite ignite, HibernateCacheProxy cache, ThreadLocal writeCtx) {
         super(ignite, cache);
 
         this.writeCtx = (ThreadLocal<WriteContext>)writeCtx;
@@ -212,7 +211,7 @@ void removed(Object key) {
          * @param cache Cache.
          * @throws IgniteCheckedException If failed.
          */
-        void updateCache(IgniteInternalCache<Object, Object> cache) throws IgniteCheckedException {
+        void updateCache(HibernateCacheProxy cache) throws IgniteCheckedException {
             if (!F.isEmpty(rmvs))
                 cache.removeAll(rmvs);
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
index e3303a71ff413..0b9a43d6a13d6 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.Query;
 import org.hibernate.cache.spi.QueryResultsRegion;
 
@@ -65,7 +64,7 @@ public class HibernateQueryResultsRegion extends HibernateGeneralDataRegion impl
      * @param cache Region cache.
      */
     public HibernateQueryResultsRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite, HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
index 58a2c4b8779d4..cdef80e0a0c0b 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.access.AccessType;
 import org.hibernate.cache.spi.access.SoftLock;
@@ -60,7 +59,7 @@ public class HibernateReadOnlyAccessStrategy extends HibernateAccessStrategyAdap
      * @param ignite Grid.
      * @param cache Cache.
      */
-    public HibernateReadOnlyAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    public HibernateReadOnlyAccessStrategy(Ignite ignite, HibernateCacheProxy cache) {
         super(ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
index bbb1d4e5dfdc2..625b05061ff0f 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
@@ -21,7 +21,6 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.transactions.Transaction;
 import org.hibernate.cache.CacheException;
@@ -68,7 +67,7 @@ public class HibernateReadWriteAccessStrategy extends HibernateAccessStrategyAda
      * @param cache Cache.
      * @param txCtx Thread local instance used to track updates done during one Hibernate transaction.
      */
-    protected HibernateReadWriteAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache, ThreadLocal txCtx) {
+    protected HibernateReadWriteAccessStrategy(Ignite ignite, HibernateCacheProxy cache, ThreadLocal txCtx) {
         super(ignite, cache);
 
         this.txCtx = (ThreadLocal<TxContext>)txCtx;
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
index 27479e90d8b7b..11a96d09096fb 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
@@ -20,7 +20,6 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.Region;
 
@@ -35,7 +34,7 @@ public class HibernateRegion implements Region {
     private final String name;
 
     /** Cache instance. */
-    protected final IgniteInternalCache<Object, Object> cache;
+    protected final HibernateCacheProxy cache;
 
     /** Grid instance. */
     protected Ignite ignite;
@@ -46,8 +45,7 @@ public class HibernateRegion implements Region {
      * @param ignite Grid.
      * @param cache Region cache.
      */
-    public HibernateRegion(HibernateRegionFactory factory, String name, Ignite ignite,
-        IgniteInternalCache<Object, Object> cache) {
+    public HibernateRegion(HibernateRegionFactory factory, String name, Ignite ignite, HibernateCacheProxy cache) {
         this.factory = factory;
         this.name = name;
         this.ignite = ignite;
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
index 825abee7e2590..4e4be36ec7c49 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.util.typedef.G;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
+import org.hibernate.cache.spi.CacheKey;
 import org.hibernate.cache.spi.CollectionRegion;
 import org.hibernate.cache.spi.EntityRegion;
 import org.hibernate.cache.spi.NaturalIdRegion;
@@ -88,7 +89,7 @@ public class HibernateRegionFactory implements RegionFactory {
     private Ignite ignite;
 
     /** Default cache. */
-    private IgniteInternalCache<Object, Object> dfltCache;
+    private HibernateCacheProxy dfltCache;
 
     /** Default region access type. */
     private AccessType dfltAccessType;
@@ -99,6 +100,23 @@ public class HibernateRegionFactory implements RegionFactory {
     /** Map needed to provide the same transaction context for different regions. */
     private final ThreadLocal threadLoc = new ThreadLocal();
 
+    /** Key transformer. */
+    private final HibernateKeyTransformer hibernate4transformer = new HibernateKeyTransformer() {
+        @Override public Object transform(Object key) {
+            if (key instanceof CacheKey) {
+                CacheKey cacheKey = (CacheKey)key;
+
+                return new HibernateKeyWrapper(
+                    cacheKey.getKey(),
+                    cacheKey.getEntityOrRoleName(),
+                    cacheKey.getTenantId()
+                );
+            }
+
+            return key;
+        }
+    };
+
     /** {@inheritDoc} */
     @Override public void start(Settings settings, Properties props) throws CacheException {
         String gridCfg = props.getProperty(GRID_CONFIG_PROPERTY);
@@ -138,10 +156,12 @@ public class HibernateRegionFactory implements RegionFactory {
         String dfltCacheName = props.getProperty(DFLT_CACHE_NAME_PROPERTY);
 
         if (dfltCacheName != null) {
-            dfltCache = ((IgniteKernal)ignite).getCache(dfltCacheName);
+            IgniteInternalCache<Object, Object> dfltCache = ((IgniteKernal)ignite).getCache(dfltCacheName);
 
             if (dfltCache == null)
                 throw new CacheException("Cache specified as default is not configured: " + dfltCacheName);
+
+            this.dfltCache = new HibernateCacheProxy(dfltCache, hibernate4transformer);
         }
 
         IgniteLogger log = ignite.log().getLogger(HibernateRegionFactory.class);
@@ -152,6 +172,7 @@ public class HibernateRegionFactory implements RegionFactory {
 
     /** {@inheritDoc} */
     @Override public void stop() {
+        // No-op.
     }
 
     /** {@inheritDoc} */
@@ -213,7 +234,7 @@ ThreadLocal threadLocalForCache(String cacheName) {
      * @return Cache for given region.
      * @throws CacheException If cache for given region is not configured.
      */
-    private IgniteInternalCache<Object, Object> regionCache(String regionName) throws CacheException {
+    private HibernateCacheProxy regionCache(String regionName) throws CacheException {
         String cacheName = regionCaches.get(regionName);
 
         if (cacheName == null) {
@@ -228,6 +249,6 @@ private IgniteInternalCache<Object, Object> regionCache(String regionName) throw
         if (cache == null)
             throw new CacheException("Cache '" + cacheName + "' for region '" + regionName + "' is not configured.");
 
-        return cache;
+        return new HibernateCacheProxy(cache, hibernate4transformer);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
index 4cedae2070e31..8b4c243277ec9 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.spi.TimestampsRegion;
 
 /**
@@ -34,7 +33,7 @@ public class HibernateTimestampsRegion extends HibernateGeneralDataRegion implem
      * @param cache Region cache.
      */
     public HibernateTimestampsRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite,  HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
index 80f75a71afed6..ca5284917aa68 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
@@ -61,7 +61,7 @@ public class HibernateTransactionalAccessStrategy extends HibernateAccessStrateg
      * @param ignite Grid.
      * @param cache Cache.
      */
-    public HibernateTransactionalAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    public HibernateTransactionalAccessStrategy(Ignite ignite, HibernateCacheProxy cache) {
         super(ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
index ed2ee01cf2280..581076a3fdbb6 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.TransactionConfiguration;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.CollectionRegion;
@@ -48,7 +47,7 @@ public class HibernateTransactionalDataRegion extends HibernateRegion implements
      * @param dataDesc Region data description.
      */
     public HibernateTransactionalDataRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache);
 
         this.dataDesc = dataDesc;
diff --git a/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java b/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java
new file mode 100644
index 0000000000000..3791baed9c93c
--- /dev/null
+++ b/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.testframework.config.GridTestProperties;
+
+/**
+ *
+ */
+public class IgniteBinaryHibernateTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception If failed.
+     */
+    public static TestSuite suite() throws Exception {
+        GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, BinaryMarshaller.class.getName());
+
+        return IgniteHibernateTestSuite.suite();
+    }
+}

From c82e25d67a2f6825a27d26933199a436f6eabba2 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dgovorukhin@gridgain.com>
Date: Fri, 7 Apr 2017 14:28:22 +0300
Subject: [PATCH 289/516] IGNITE-4889 - Changed Hibernate integration to use
 custom keys

(cherry picked from commit 6b62a20)
---
 .../HibernateAccessStrategyAdapter.java       |   8 +-
 .../cache/hibernate/HibernateCacheProxy.java  | 818 ++++++++++++++++++
 .../hibernate/HibernateCollectionRegion.java  |   3 +-
 .../hibernate/HibernateEntityRegion.java      |   3 +-
 .../hibernate/HibernateGeneralDataRegion.java |   3 +-
 .../hibernate/HibernateKeyTransformer.java    |  29 +
 .../cache/hibernate/HibernateKeyWrapper.java  |  72 ++
 .../hibernate/HibernateNaturalIdRegion.java   |   3 +-
 .../HibernateNonStrictAccessStrategy.java     |   5 +-
 .../HibernateQueryResultsRegion.java          |   3 +-
 .../HibernateReadOnlyAccessStrategy.java      |   3 +-
 .../HibernateReadWriteAccessStrategy.java     |   3 +-
 .../cache/hibernate/HibernateRegion.java      |   6 +-
 .../hibernate/HibernateRegionFactory.java     |  29 +-
 .../hibernate/HibernateTimestampsRegion.java  |   3 +-
 .../HibernateTransactionalAccessStrategy.java |   2 +-
 .../HibernateTransactionalDataRegion.java     |   3 +-
 .../IgniteBinaryHibernateTestSuite.java       |  37 +
 18 files changed, 1000 insertions(+), 33 deletions(-)
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
 create mode 100644 modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
 create mode 100644 modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java

diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
index 27734d9ad9ca5..f6c1d0e15eb43 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateAccessStrategyAdapter.java
@@ -92,7 +92,7 @@
  */
 public abstract class HibernateAccessStrategyAdapter {
     /** */
-    protected final IgniteInternalCache<Object, Object> cache;
+    protected final HibernateCacheProxy cache;
 
     /** Grid. */
     protected final Ignite ignite;
@@ -104,7 +104,7 @@ public abstract class HibernateAccessStrategyAdapter {
      * @param ignite Grid.
      * @param cache Cache.
      */
-    protected HibernateAccessStrategyAdapter(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    protected HibernateAccessStrategyAdapter(Ignite ignite, HibernateCacheProxy cache) {
         this.cache = cache;
         this.ignite = ignite;
 
@@ -292,8 +292,10 @@ protected final void removeAll() throws CacheException {
      * @param key Key.
      * @throws CacheException If failed.
      */
-    static void evict(Ignite ignite, IgniteInternalCache<Object,Object> cache, Object key) throws CacheException {
+    static void evict(Ignite ignite, HibernateCacheProxy cache, Object key) throws CacheException {
         try {
+            key = cache.keyTransformer().transform(key);
+
             ignite.compute(ignite.cluster()).call(new ClearKeyCallable(key, cache.name()));
         }
         catch (IgniteException e) {
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
new file mode 100644
index 0000000000000..871c4a100ac6f
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
@@ -0,0 +1,818 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorResult;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheEntry;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.mxbean.CacheMetricsMXBean;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Hibernate cache proxy.
+ */
+public class HibernateCacheProxy implements IgniteInternalCache<Object, Object> {
+    /** Delegate. */
+    private final IgniteInternalCache<Object, Object> delegate;
+
+    /** Transformer. */
+    private final HibernateKeyTransformer keyTransformer;
+
+    /**
+     * @param delegate Delegate.
+     * @param keyTransformer Key keyTransformer.
+     */
+    HibernateCacheProxy(
+        IgniteInternalCache<Object, Object> delegate,
+        HibernateKeyTransformer keyTransformer
+    ) {
+        assert delegate != null;
+        assert keyTransformer != null;
+
+        this.delegate = delegate;
+        this.keyTransformer = keyTransformer;
+    }
+
+    /**
+     * @return HibernateKeyTransformer
+     */
+    HibernateKeyTransformer keyTransformer(){
+        return keyTransformer;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String name() {
+        return delegate.name();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean skipStore() {
+        return delegate.skipStore();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache setSkipStore(boolean skipStore) {
+        return delegate.setSkipStore(skipStore);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean containsKey(Object key) {
+        return delegate.containsKey(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> containsKeyAsync(Object key) {
+        return delegate.containsKeyAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void promoteAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.promoteAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public long overflowSize() throws IgniteCheckedException {
+        return delegate.overflowSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long swapSize() throws IgniteCheckedException {
+        return delegate.swapSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long swapKeys() throws IgniteCheckedException {
+        return delegate.swapKeys();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean containsKeys(Collection keys) {
+        return delegate.containsKey(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> containsKeysAsync(Collection keys) {
+        return delegate.containsKeysAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object localPeek(
+        Object key,
+        CachePeekMode[] peekModes,
+        @Nullable IgniteCacheExpiryPolicy plc
+    ) throws IgniteCheckedException {
+        return delegate.localPeek(keyTransformer.transform(key), peekModes, plc);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable<Cache.Entry<Object, Object>> localEntries(
+        CachePeekMode[] peekModes
+    ) throws IgniteCheckedException {
+        return delegate.localEntries(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object get(Object key) throws IgniteCheckedException {
+        return delegate.get(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public CacheEntry getEntry(Object key) throws IgniteCheckedException {
+        return delegate.getEntry(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAsync(Object key) {
+        return delegate.getAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<CacheEntry<Object, Object>> getEntryAsync(Object key) {
+        return delegate.getEntryAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map getAll(@Nullable Collection keys) throws IgniteCheckedException {
+        return delegate.getAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<CacheEntry<Object, Object>> getEntries(
+        @Nullable Collection keys) throws IgniteCheckedException {
+        return delegate.getEntries(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map<Object, Object>> getAllAsync(@Nullable Collection keys) {
+        return delegate.getAllAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Collection<CacheEntry<Object,Object>>> getEntriesAsync(
+        @Nullable Collection keys
+    ) {
+        return delegate.getEntriesAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndPut(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndPut(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndPutAsync(Object key, Object val) {
+        return delegate.getAndPutAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean put(Object key, Object val) throws IgniteCheckedException {
+        return delegate.put(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> putAsync(Object key, Object val) {
+        return delegate.putAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndPutIfAbsent(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndPutIfAbsent(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndPutIfAbsentAsync(Object key, Object val) {
+        return delegate.getAndPutIfAbsentAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean putIfAbsent(Object key, Object val) throws IgniteCheckedException {
+        return delegate.putIfAbsent(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(Object key, Object val) {
+        return delegate.putIfAbsentAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndReplace(Object key, Object val) throws IgniteCheckedException {
+        return delegate.getAndReplace(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndReplaceAsync(Object key, Object val) {
+        return delegate.getAndReplaceAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean replace(Object key, Object val) throws IgniteCheckedException {
+        return delegate.replace(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> replaceAsync(Object key, Object val) {
+        return delegate.replaceAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean replace(Object key, Object oldVal, Object newVal) throws IgniteCheckedException {
+        return delegate.replace(keyTransformer.transform(key), oldVal, newVal);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> replaceAsync(Object key, Object oldVal, Object newVal) {
+        return delegate.replaceAsync(keyTransformer.transform(key), oldVal, newVal);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void putAll(@Nullable Map m) throws IgniteCheckedException {
+        delegate.putAll(transform(m));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> putAllAsync(@Nullable Map m) {
+        return delegate.putAllAsync(transform(m));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set keySet() {
+        return delegate.keySet();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set keySetx() {
+        return delegate.keySetx();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set primaryKeySet() {
+        return delegate.primaryKeySet();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable values() {
+        return delegate.values();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set<Cache.Entry<Object, Object>> entrySet() {
+        return delegate.entrySet();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Set<Cache.Entry<Object,Object>> entrySet(int part) {
+        return delegate.entrySet(part);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Set<Cache.Entry<Object, Object>> entrySetx(CacheEntryPredicate... filter) {
+        return delegate.entrySetx(filter);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Transaction txStart(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation
+    ) {
+        return delegate.txStart(concurrency, isolation);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalTx txStartEx(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation
+    ) {
+        return delegate.txStartEx(concurrency, isolation);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Transaction txStart(
+        TransactionConcurrency concurrency,
+        TransactionIsolation isolation,
+        long timeout,
+        int txSize
+    ) {
+        return delegate.txStart(concurrency, isolation, timeout, txSize);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Transaction tx() {
+        return delegate.tx();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean evict(Object key) {
+        return delegate.evict(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void evictAll(@Nullable Collection keys) {
+        delegate.evictAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearLocally(boolean srv, boolean near, boolean readers) {
+        delegate.clearLocally(srv, near, readers);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean clearLocally(Object key) {
+        return delegate.clearLocally(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearLocallyAll(Set keys, boolean srv, boolean near, boolean readers) {
+        delegate.clearLocallyAll((Set<?>)transform(keys), srv, near, readers);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear(Object key) throws IgniteCheckedException {
+        delegate.clear(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clearAll(Set keys) throws IgniteCheckedException {
+        delegate.clearAll((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear() throws IgniteCheckedException {
+        delegate.clear();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAsync() {
+        return delegate.clearAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAsync(Object key) {
+        return delegate.clearAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> clearAllAsync(Set keys) {
+        return delegate.clearAllAsync((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getAndRemove(Object key) throws IgniteCheckedException {
+        return delegate.getAndRemove(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getAndRemoveAsync(Object key) {
+        return delegate.getAndRemoveAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(Object key) throws IgniteCheckedException {
+        return delegate.remove(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> removeAsync(Object key) {
+        return delegate.removeAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(Object key, Object val) throws IgniteCheckedException {
+        return delegate.remove(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> removeAsync(Object key, Object val) {
+        return delegate.removeAsync(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.removeAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllAsync(@Nullable Collection keys) {
+        return delegate.removeAllAsync(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAll() throws IgniteCheckedException {
+        delegate.removeAll();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllAsync() {
+        return delegate.removeAllAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lock(Object key, long timeout) throws IgniteCheckedException {
+        return delegate.lock(keyTransformer.transform(key), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> lockAsync(Object key, long timeout) {
+        return delegate.lockAsync(keyTransformer.transform(key), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lockAll(@Nullable Collection keys, long timeout) throws IgniteCheckedException {
+        return delegate.lockAll(transform(keys), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Boolean> lockAllAsync(@Nullable Collection keys, long timeout) {
+        return delegate.lockAllAsync(transform(keys), timeout);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlock(Object key) throws IgniteCheckedException {
+        delegate.unlock(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlockAll(@Nullable Collection keys) throws IgniteCheckedException {
+        delegate.unlockAll(transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLocked(Object key) {
+        return delegate.isLocked(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLockedByThread(Object key) {
+        return delegate.isLockedByThread(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size() {
+        return delegate.size();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong() {
+        return delegate.sizeLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int localSize(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSize(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long localSizeLong(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSizeLong(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long localSizeLong(int partition, CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.localSizeLong(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.size(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong(CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.sizeLong(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long sizeLong(int partition, CachePeekMode[] peekModes) throws IgniteCheckedException {
+        return delegate.sizeLong(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Integer> sizeAsync(CachePeekMode[] peekModes) {
+        return delegate.sizeAsync(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Long> sizeLongAsync(CachePeekMode[] peekModes) {
+        return delegate.sizeLongAsync(peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Long> sizeLongAsync(int partition, CachePeekMode[] peekModes) {
+        return delegate.sizeLongAsync(partition, peekModes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int nearSize() {
+        return delegate.nearSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int primarySize() {
+        return delegate.primarySize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long primarySizeLong() {
+        return delegate.primarySizeLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheConfiguration configuration() {
+        return delegate.configuration();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Affinity affinity() {
+        return delegate.affinity();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics clusterMetrics() {
+        return delegate.clusterMetrics();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics clusterMetrics(ClusterGroup grp) {
+        return delegate.clusterMetrics(grp);
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetrics localMetrics() {
+        return delegate.localMetrics();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetricsMXBean clusterMxBean() {
+        return delegate.clusterMxBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public CacheMetricsMXBean localMxBean() {
+        return delegate.localMxBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long offHeapEntriesCount() {
+        return delegate.offHeapEntriesCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long offHeapAllocatedSize() {
+        return delegate.offHeapAllocatedSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> rebalance() {
+        return delegate.rebalance();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache forSubjectId(UUID subjId) {
+        return delegate.forSubjectId(subjId);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object getForcePrimary(Object key) throws IgniteCheckedException {
+        return delegate.getForcePrimary(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture getForcePrimaryAsync(Object key) {
+        return delegate.getForcePrimaryAsync(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map getAllOutTx(Set keys) throws IgniteCheckedException {
+        return delegate.getAllOutTx((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map<Object, Object>> getAllOutTxAsync(Set keys) {
+        return delegate.getAllOutTxAsync((Set<?>)transform(keys));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isIgfsDataCache() {
+        return delegate.isIgfsDataCache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long igfsDataSpaceUsed() {
+        return delegate.igfsDataSpaceUsed();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long igfsDataSpaceMax() {
+        return delegate.igfsDataSpaceMax();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMongoDataCache() {
+        return delegate.isMongoDataCache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMongoMetaCache() {
+        return delegate.isMongoMetaCache();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public ExpiryPolicy expiry() {
+        return delegate.expiry();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache withExpiryPolicy(ExpiryPolicy plc) {
+        return delegate.withExpiryPolicy(plc);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache withNoRetries() {
+        return delegate.withNoRetries();
+    }
+
+    /** {@inheritDoc} */
+    @Override public GridCacheContext context() {
+        return delegate.context();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void localLoadCache(
+        @Nullable IgniteBiPredicate p,
+        @Nullable Object... args
+    ) throws IgniteCheckedException {
+        delegate.localLoadCache(p, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> localLoadCacheAsync(
+        @Nullable IgniteBiPredicate p,
+        @Nullable Object... args
+    ) {
+        return delegate.localLoadCacheAsync(p, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Object getTopologySafe(Object key) throws IgniteCheckedException {
+        return delegate.getTopologySafe(keyTransformer.transform(key));
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object tryGetAndPut(Object key, Object val) throws IgniteCheckedException {
+        return delegate.tryGetAndPut(keyTransformer.transform(key), val);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryProcessorResult invoke(
+        @Nullable AffinityTopologyVersion topVer,
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) throws IgniteCheckedException {
+        return delegate.invoke(topVer, key, entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map> invokeAllAsync(Map map, Object... args) {
+        return delegate.invokeAllAsync(map, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map invokeAll(Map map, Object... args) throws IgniteCheckedException {
+        return delegate.invokeAll(map, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<Map> invokeAllAsync(Set keys, EntryProcessor entryProcessor, Object... args) {
+        return delegate.invokeAllAsync((Set<?>)transform(keys), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map invokeAll(Set keys, EntryProcessor entryProcessor, Object... args) throws IgniteCheckedException {
+        return delegate.invokeAll((Set<?>)transform(keys), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<EntryProcessorResult> invokeAsync(
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) {
+        return delegate.invokeAsync(keyTransformer.transform(key), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public EntryProcessorResult invoke(
+        Object key,
+        EntryProcessor entryProcessor,
+        Object... args
+    ) throws IgniteCheckedException {
+        return delegate.invoke(keyTransformer.transform(key), entryProcessor, args);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> removeAllConflictAsync(Map drMap) throws IgniteCheckedException {
+        return delegate.removeAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeAllConflict(Map drMap) throws IgniteCheckedException {
+        delegate.removeAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> putAllConflictAsync(Map drMap) throws IgniteCheckedException {
+        return delegate.putAllConflictAsync(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void putAllConflict(Map drMap) throws IgniteCheckedException {
+        delegate.putAllConflict(drMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache keepBinary() {
+        return delegate.keepBinary();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalCache cache() {
+        return delegate.cache();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterator iterator() {
+        return delegate.iterator();
+    }
+
+    /**
+     * @param keys Keys.
+     */
+    private Collection<Object> transform(Collection<Object> keys) {
+        Collection<Object> res = new LinkedList<>();
+
+        for (Object o : keys)
+            res.add(keyTransformer.transform(o));
+
+        return res;
+    }
+
+    /**
+     * @param map Map.
+     */
+    private Map<Object, Object> transform(Map<Object, Object> map) {
+        Map<Object, Object> res = new HashMap<>();
+
+        Set<Map.Entry<Object, Object>> ents = map.entrySet();
+
+        for (Map.Entry<Object, Object> e : ents)
+            res.put(keyTransformer.transform(e.getKey()), e.getValue());
+
+        return res;
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
index 045f401d92f80..eb35a2c4709d7 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCollectionRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.CollectionRegion;
@@ -72,7 +71,7 @@ public class HibernateCollectionRegion extends HibernateTransactionalDataRegion
      * @param dataDesc Region data description.
      */
     public HibernateCollectionRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
index 1ceda144a8355..ad5b1919f51b3 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateEntityRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.EntityRegion;
@@ -62,7 +61,7 @@ public class HibernateEntityRegion extends HibernateTransactionalDataRegion impl
      * @param dataDesc Region data description.
      */
     public HibernateEntityRegion(HibernateRegionFactory factory, String name, Ignite ignite,
-        IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
index fbac624f66af1..2f1a11dc8baa3 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateGeneralDataRegion.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.GeneralDataRegion;
 import org.hibernate.cache.spi.QueryResultsRegion;
@@ -38,7 +37,7 @@ public class HibernateGeneralDataRegion extends HibernateRegion implements Gener
      * @param cache Region cache.
      */
     public HibernateGeneralDataRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite, HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
new file mode 100644
index 0000000000000..97fc0e9eb15e6
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyTransformer.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+/**
+ * An interface for transforming hibernate keys to Ignite keys.
+ */
+public interface HibernateKeyTransformer {
+    /**
+     * @param key Hibernate key.
+     * @return Transformed key.
+     */
+    public Object transform(Object key);
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
new file mode 100644
index 0000000000000..7de440ebb032c
--- /dev/null
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateKeyWrapper.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.hibernate;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Hibernate cache key wrapper.
+ */
+public class HibernateKeyWrapper {
+    /** Key. */
+    private final Object key;
+
+    /** Entry. */
+    private final String entry;
+
+    /** */
+    private final String tenantId;
+
+    /**
+     * @param key Key.
+     * @param entry Entry.
+     * @param tenantId Tenant ID.
+     */
+    HibernateKeyWrapper(Object key, String entry, String tenantId) {
+        this.key = key;
+        this.entry = entry;
+        this.tenantId = tenantId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o) return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        HibernateKeyWrapper that = (HibernateKeyWrapper) o;
+
+        return (key != null ? key.equals(that.key) : that.key == null) &&
+            (entry != null ? entry.equals(that.entry) : that.entry == null) &&
+            (tenantId != null ? tenantId.equals(that.tenantId) : that.tenantId == null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = key != null ? key.hashCode() : 0;
+        res = 31 * res + (entry != null ? entry.hashCode() : 0);
+        res = 31 * res + (tenantId != null ? tenantId.hashCode() : 0);
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HibernateKeyWrapper.class, this);
+    }
+}
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
index 99d5348dea479..862a4228f2767 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNaturalIdRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.NaturalIdRegion;
@@ -52,7 +51,7 @@ public class HibernateNaturalIdRegion extends HibernateTransactionalDataRegion i
      * @param dataDesc Region data description.
      */
     public HibernateNaturalIdRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache, dataDesc);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
index 1cb8d48e0be2f..a36d7e786d739 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateNonStrictAccessStrategy.java
@@ -21,7 +21,6 @@
 import java.util.Set;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridLeanMap;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.typedef.F;
@@ -67,7 +66,7 @@ public class HibernateNonStrictAccessStrategy extends HibernateAccessStrategyAda
      * @param cache Cache.
      * @param writeCtx Thread local instance used to track updates done during one Hibernate transaction.
      */
-    protected HibernateNonStrictAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache, ThreadLocal writeCtx) {
+    protected HibernateNonStrictAccessStrategy(Ignite ignite, HibernateCacheProxy cache, ThreadLocal writeCtx) {
         super(ignite, cache);
 
         this.writeCtx = (ThreadLocal<WriteContext>)writeCtx;
@@ -212,7 +211,7 @@ void removed(Object key) {
          * @param cache Cache.
          * @throws IgniteCheckedException If failed.
          */
-        void updateCache(IgniteInternalCache<Object, Object> cache) throws IgniteCheckedException {
+        void updateCache(HibernateCacheProxy cache) throws IgniteCheckedException {
             if (!F.isEmpty(rmvs))
                 cache.removeAll(rmvs);
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
index e3303a71ff413..0b9a43d6a13d6 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateQueryResultsRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.Query;
 import org.hibernate.cache.spi.QueryResultsRegion;
 
@@ -65,7 +64,7 @@ public class HibernateQueryResultsRegion extends HibernateGeneralDataRegion impl
      * @param cache Region cache.
      */
     public HibernateQueryResultsRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite, HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
index 58a2c4b8779d4..cdef80e0a0c0b 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadOnlyAccessStrategy.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.access.AccessType;
 import org.hibernate.cache.spi.access.SoftLock;
@@ -60,7 +59,7 @@ public class HibernateReadOnlyAccessStrategy extends HibernateAccessStrategyAdap
      * @param ignite Grid.
      * @param cache Cache.
      */
-    public HibernateReadOnlyAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    public HibernateReadOnlyAccessStrategy(Ignite ignite, HibernateCacheProxy cache) {
         super(ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
index bbb1d4e5dfdc2..625b05061ff0f 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateReadWriteAccessStrategy.java
@@ -21,7 +21,6 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.transactions.Transaction;
 import org.hibernate.cache.CacheException;
@@ -68,7 +67,7 @@ public class HibernateReadWriteAccessStrategy extends HibernateAccessStrategyAda
      * @param cache Cache.
      * @param txCtx Thread local instance used to track updates done during one Hibernate transaction.
      */
-    protected HibernateReadWriteAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache, ThreadLocal txCtx) {
+    protected HibernateReadWriteAccessStrategy(Ignite ignite, HibernateCacheProxy cache, ThreadLocal txCtx) {
         super(ignite, cache);
 
         this.txCtx = (ThreadLocal<TxContext>)txCtx;
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
index 27479e90d8b7b..11a96d09096fb 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegion.java
@@ -20,7 +20,6 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.Region;
 
@@ -35,7 +34,7 @@ public class HibernateRegion implements Region {
     private final String name;
 
     /** Cache instance. */
-    protected final IgniteInternalCache<Object, Object> cache;
+    protected final HibernateCacheProxy cache;
 
     /** Grid instance. */
     protected Ignite ignite;
@@ -46,8 +45,7 @@ public class HibernateRegion implements Region {
      * @param ignite Grid.
      * @param cache Region cache.
      */
-    public HibernateRegion(HibernateRegionFactory factory, String name, Ignite ignite,
-        IgniteInternalCache<Object, Object> cache) {
+    public HibernateRegion(HibernateRegionFactory factory, String name, Ignite ignite, HibernateCacheProxy cache) {
         this.factory = factory;
         this.name = name;
         this.ignite = ignite;
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
index 825abee7e2590..4e4be36ec7c49 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateRegionFactory.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.util.typedef.G;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
+import org.hibernate.cache.spi.CacheKey;
 import org.hibernate.cache.spi.CollectionRegion;
 import org.hibernate.cache.spi.EntityRegion;
 import org.hibernate.cache.spi.NaturalIdRegion;
@@ -88,7 +89,7 @@ public class HibernateRegionFactory implements RegionFactory {
     private Ignite ignite;
 
     /** Default cache. */
-    private IgniteInternalCache<Object, Object> dfltCache;
+    private HibernateCacheProxy dfltCache;
 
     /** Default region access type. */
     private AccessType dfltAccessType;
@@ -99,6 +100,23 @@ public class HibernateRegionFactory implements RegionFactory {
     /** Map needed to provide the same transaction context for different regions. */
     private final ThreadLocal threadLoc = new ThreadLocal();
 
+    /** Key transformer. */
+    private final HibernateKeyTransformer hibernate4transformer = new HibernateKeyTransformer() {
+        @Override public Object transform(Object key) {
+            if (key instanceof CacheKey) {
+                CacheKey cacheKey = (CacheKey)key;
+
+                return new HibernateKeyWrapper(
+                    cacheKey.getKey(),
+                    cacheKey.getEntityOrRoleName(),
+                    cacheKey.getTenantId()
+                );
+            }
+
+            return key;
+        }
+    };
+
     /** {@inheritDoc} */
     @Override public void start(Settings settings, Properties props) throws CacheException {
         String gridCfg = props.getProperty(GRID_CONFIG_PROPERTY);
@@ -138,10 +156,12 @@ public class HibernateRegionFactory implements RegionFactory {
         String dfltCacheName = props.getProperty(DFLT_CACHE_NAME_PROPERTY);
 
         if (dfltCacheName != null) {
-            dfltCache = ((IgniteKernal)ignite).getCache(dfltCacheName);
+            IgniteInternalCache<Object, Object> dfltCache = ((IgniteKernal)ignite).getCache(dfltCacheName);
 
             if (dfltCache == null)
                 throw new CacheException("Cache specified as default is not configured: " + dfltCacheName);
+
+            this.dfltCache = new HibernateCacheProxy(dfltCache, hibernate4transformer);
         }
 
         IgniteLogger log = ignite.log().getLogger(HibernateRegionFactory.class);
@@ -152,6 +172,7 @@ public class HibernateRegionFactory implements RegionFactory {
 
     /** {@inheritDoc} */
     @Override public void stop() {
+        // No-op.
     }
 
     /** {@inheritDoc} */
@@ -213,7 +234,7 @@ ThreadLocal threadLocalForCache(String cacheName) {
      * @return Cache for given region.
      * @throws CacheException If cache for given region is not configured.
      */
-    private IgniteInternalCache<Object, Object> regionCache(String regionName) throws CacheException {
+    private HibernateCacheProxy regionCache(String regionName) throws CacheException {
         String cacheName = regionCaches.get(regionName);
 
         if (cacheName == null) {
@@ -228,6 +249,6 @@ private IgniteInternalCache<Object, Object> regionCache(String regionName) throw
         if (cache == null)
             throw new CacheException("Cache '" + cacheName + "' for region '" + regionName + "' is not configured.");
 
-        return cache;
+        return new HibernateCacheProxy(cache, hibernate4transformer);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
index 4cedae2070e31..8b4c243277ec9 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTimestampsRegion.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.cache.hibernate;
 
 import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.spi.TimestampsRegion;
 
 /**
@@ -34,7 +33,7 @@ public class HibernateTimestampsRegion extends HibernateGeneralDataRegion implem
      * @param cache Region cache.
      */
     public HibernateTimestampsRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+        Ignite ignite,  HibernateCacheProxy cache) {
         super(factory, name, ignite, cache);
     }
 }
\ No newline at end of file
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
index 80f75a71afed6..ca5284917aa68 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalAccessStrategy.java
@@ -61,7 +61,7 @@ public class HibernateTransactionalAccessStrategy extends HibernateAccessStrateg
      * @param ignite Grid.
      * @param cache Cache.
      */
-    public HibernateTransactionalAccessStrategy(Ignite ignite, IgniteInternalCache<Object, Object> cache) {
+    public HibernateTransactionalAccessStrategy(Ignite ignite, HibernateCacheProxy cache) {
         super(ignite, cache);
     }
 
diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
index ed2ee01cf2280..581076a3fdbb6 100644
--- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
+++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateTransactionalDataRegion.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.TransactionConfiguration;
-import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.hibernate.cache.CacheException;
 import org.hibernate.cache.spi.CacheDataDescription;
 import org.hibernate.cache.spi.CollectionRegion;
@@ -48,7 +47,7 @@ public class HibernateTransactionalDataRegion extends HibernateRegion implements
      * @param dataDesc Region data description.
      */
     public HibernateTransactionalDataRegion(HibernateRegionFactory factory, String name,
-        Ignite ignite, IgniteInternalCache<Object, Object> cache, CacheDataDescription dataDesc) {
+        Ignite ignite, HibernateCacheProxy cache, CacheDataDescription dataDesc) {
         super(factory, name, ignite, cache);
 
         this.dataDesc = dataDesc;
diff --git a/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java b/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java
new file mode 100644
index 0000000000000..3791baed9c93c
--- /dev/null
+++ b/modules/hibernate/src/test/java/org/apache/ignite/testsuites/IgniteBinaryHibernateTestSuite.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.testframework.config.GridTestProperties;
+
+/**
+ *
+ */
+public class IgniteBinaryHibernateTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception If failed.
+     */
+    public static TestSuite suite() throws Exception {
+        GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, BinaryMarshaller.class.getName());
+
+        return IgniteHibernateTestSuite.suite();
+    }
+}

From a352951d91edde9c0029a8bf435d61b4a7cd8c11 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 4 Jul 2017 20:24:52 +0300
Subject: [PATCH 290/516] IGNITE-4831: Add an option to disable MBeans.

---
 .../apache/ignite/IgniteSystemProperties.java |   8 ++
 .../org/apache/ignite/cache/CacheManager.java |  11 ++
 .../apache/ignite/internal/IgniteKernal.java  |  35 +++--
 .../apache/ignite/internal/IgnitionEx.java    |  10 +-
 .../client/router/impl/GridTcpRouterImpl.java |  57 ++++++---
 .../processors/cache/GridCacheProcessor.java  |  10 +-
 .../ignite/internal/util/IgniteUtils.java     |  17 ++-
 .../apache/ignite/spi/IgniteSpiAdapter.java   |   3 +
 .../util/mbeans/GridMBeanDisableSelfTest.java | 121 ++++++++++++++++++
 9 files changed, 238 insertions(+), 34 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index c4208a76f1a7d..e38d236f49964 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -382,6 +382,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_MBEAN_APPEND_CLASS_LOADER_ID = "IGNITE_MBEAN_APPEND_CLASS_LOADER_ID";
 
+    /**
+     * If property is set to {@code true}, then Ignite will disable MBeans registration.
+     * This may be helpful if MBeans are not allowed e.g. for security reasons.
+     *
+     * Default is {@code false}
+     */
+    public static final String IGNITE_MBEANS_DISABLED = "IGNITE_MBEANS_DISABLED";
+
     /**
      * Property controlling size of buffer holding last exception. Default value of {@code 1000}.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java
index 6e09d7276f3ad..a5e612ef986fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java
@@ -44,7 +44,9 @@
 import org.apache.ignite.internal.IgnitionEx;
 import org.apache.ignite.internal.mxbean.IgniteStandardMXBean;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
+import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.jetbrains.annotations.Nullable;
@@ -314,6 +316,9 @@ private ObjectName getObjectName(String cacheName, String objName) {
 
     /** {@inheritDoc} */
     @Override public void enableManagement(String cacheName, boolean enabled) {
+        if(IgniteUtils.IGNITE_MBEANS_DISABLED)
+            return;
+
         kernalGateway.readLock();
 
         try {
@@ -336,6 +341,9 @@ private ObjectName getObjectName(String cacheName, String objName) {
 
     /** {@inheritDoc} */
     @Override public void enableStatistics(String cacheName, boolean enabled) {
+        if(IgniteUtils.IGNITE_MBEANS_DISABLED)
+            return;
+
         kernalGateway.readLock();
 
         try {
@@ -389,6 +397,9 @@ private void registerCacheObject(Object mxbean, String name, String beanType) {
      * @param beanType Mxbean name.
      */
     private void unregisterCacheObject(String name, String beanType) {
+        if(IgniteUtils.IGNITE_MBEANS_DISABLED)
+            return;
+
         MBeanServer mBeanSrv = ignite.configuration().getMBeanServer();
 
         Set<ObjectName> registeredObjNames = mBeanSrv.queryNames(getObjectName(name, beanType), null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 2aa05f1b9ab11..9bada6c35ce3b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1494,6 +1494,9 @@ private void addSpiAttributes(IgniteSpi... spiList) throws IgniteCheckedExceptio
 
     /** @throws IgniteCheckedException If registration failed. */
     private void registerKernalMBean() throws IgniteCheckedException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
+
         try {
             kernalMBean = U.registerMBean(
                 cfg.getMBeanServer(),
@@ -1515,6 +1518,9 @@ private void registerKernalMBean() throws IgniteCheckedException {
 
     /** @throws IgniteCheckedException If registration failed. */
     private void registerLocalNodeMBean() throws IgniteCheckedException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
+
         ClusterLocalNodeMetricsMXBean mbean = new ClusterLocalNodeMetricsMXBeanImpl(ctx.discovery().localNode());
 
         try {
@@ -1546,6 +1552,8 @@ private void registerExecutorMBeans(
         ExecutorService utilityExecSvc,
         ExecutorService marshallerExecSvc
     ) throws IgniteCheckedException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
         pubExecSvcMBean = registerExecutorMBean(execSvc, "GridExecutionExecutor");
         sysExecSvcMBean = registerExecutorMBean(sysExecSvc, "GridSystemExecutor");
         mgmtExecSvcMBean = registerExecutorMBean(mgmtExecSvc, "GridManagementExecutor");
@@ -1567,6 +1575,7 @@ private void registerExecutorMBeans(
      */
     private ObjectName registerExecutorMBean(ExecutorService exec, String name) throws IgniteCheckedException {
         assert exec != null;
+        assert !U.IGNITE_MBEANS_DISABLED;
 
         try {
             ObjectName res = U.registerMBean(
@@ -1595,22 +1604,24 @@ private ObjectName registerExecutorMBean(ExecutorService exec, String name) thro
      * @return {@code True} if successfully unregistered, {@code false} otherwise.
      */
     private boolean unregisterMBean(@Nullable ObjectName mbean) {
-        if (mbean != null)
-            try {
-                cfg.getMBeanServer().unregisterMBean(mbean);
+        if (mbean == null)
+            return true;
 
-                if (log.isDebugEnabled())
-                    log.debug("Unregistered MBean: " + mbean);
+        assert !U.IGNITE_MBEANS_DISABLED;
 
-                return true;
-            }
-            catch (JMException e) {
-                U.error(log, "Failed to unregister MBean.", e);
+        try {
+            cfg.getMBeanServer().unregisterMBean(mbean);
 
-                return false;
-            }
+            if (log.isDebugEnabled())
+                log.debug("Unregistered MBean: " + mbean);
+
+            return true;
+        }
+        catch (JMException e) {
+            U.error(log, "Failed to unregister MBean.", e);
 
-        return true;
+            return false;
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 5b2c3fc53faea..8d54ba57b8762 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -1950,7 +1950,7 @@ private IgniteConfiguration initializeConfiguration(IgniteConfiguration cfg)
             if (myCfg.getUserAttributes() == null)
                 myCfg.setUserAttributes(Collections.<String, Object>emptyMap());
 
-            if (myCfg.getMBeanServer() == null)
+            if (myCfg.getMBeanServer() == null && !U.IGNITE_MBEANS_DISABLED)
                 myCfg.setMBeanServer(ManagementFactory.getPlatformMBeanServer());
 
             Marshaller marsh = myCfg.getMarshaller();
@@ -2430,6 +2430,11 @@ private void stopExecutors0(IgniteLogger log) {
          * @throws IgniteCheckedException If registration failed.
          */
         private void registerFactoryMbean(MBeanServer srv) throws IgniteCheckedException {
+            if(U.IGNITE_MBEANS_DISABLED)
+                return;
+
+            assert srv != null;
+
             synchronized (mbeans) {
                 GridMBeanServerData data = mbeans.get(srv);
 
@@ -2480,6 +2485,9 @@ private void registerFactoryMbean(MBeanServer srv) throws IgniteCheckedException
          * Unregister delegate Mbean instance for {@link Ignition}.
          */
         private void unregisterFactoryMBean() {
+            if(U.IGNITE_MBEANS_DISABLED)
+                return;
+
             synchronized (mbeans) {
                 Iterator<Entry<MBeanServer, GridMBeanServerData>> iter = mbeans.entrySet().iterator();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
index 06a492924f408..6210bb06b765b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
@@ -169,6 +169,32 @@ public GridTcpRouterImpl(GridTcpRouterConfiguration cfg) {
                 "are in use) [firstPort=" + cfg.getPort() + ", lastPort=" + (cfg.getPort() + cfg.getPortRange()) +
                 ", addr=" + hostAddr + ']');
 
+        registerMBean();
+    }
+
+    /**
+     * Stops this router.
+     */
+    @Override public void stop() {
+        if (srv != null)
+            srv.stop();
+
+        if (client != null)
+            client.stop(true);
+
+        unregisterMBean();
+
+        if (log.isInfoEnabled())
+            log.info("TCP router successfully stopped.");
+    }
+
+    /**
+     * Try to register MBean.
+     */
+    private void registerMBean() {
+        if (U.IGNITE_MBEANS_DISABLED)
+            return;
+
         try {
             ObjectName objName = U.registerMBean(
                 ManagementFactory.getPlatformMBeanServer(),
@@ -189,28 +215,23 @@ public GridTcpRouterImpl(GridTcpRouterConfiguration cfg) {
     }
 
     /**
-     * Stops this router.
+     * Unregister MBean.
      */
-    @Override public void stop() {
-        if (srv != null)
-            srv.stop();
-
-        if (client != null)
-            client.stop(true);
+    private void unregisterMBean() {
+        if (mbeanName == null)
+            return;
 
-        if (mbeanName != null)
-            try {
-                ManagementFactory.getPlatformMBeanServer().unregisterMBean(mbeanName);
+        assert !U.IGNITE_MBEANS_DISABLED;
 
-                if (log.isDebugEnabled())
-                    log.debug("Unregistered MBean: " + mbeanName);
-            }
-            catch (JMException e) {
-                U.error(log, "Failed to unregister MBean.", e);
-            }
+        try {
+            ManagementFactory.getPlatformMBeanServer().unregisterMBean(mbeanName);
 
-        if (log.isInfoEnabled())
-            log.info("TCP router successfully stopped.");
+            if (log.isDebugEnabled())
+                log.debug("Unregistered MBean: " + mbeanName);
+        }
+        catch (JMException e) {
+            U.error(log, "Failed to unregister MBean.", e);
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index ccd7ae0bf500d..341b2c19d38e4 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3497,6 +3497,9 @@ public void createMissingCaches() throws IgniteCheckedException {
     @SuppressWarnings("unchecked")
     private void registerMbean(Object o, @Nullable String cacheName, boolean near)
         throws IgniteCheckedException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
+
         assert o != null;
 
         MBeanServer srvr = ctx.config().getMBeanServer();
@@ -3513,7 +3516,7 @@ private void registerMbean(Object o, @Nullable String cacheName, boolean near)
                     U.registerCacheMBean(srvr, ctx.gridName(), cacheName, o.getClass().getName(), o,
                         (Class<Object>)itf);
                 }
-                catch (JMException e) {
+                catch (Throwable e) {
                     throw new IgniteCheckedException("Failed to register MBean for component: " + o, e);
                 }
 
@@ -3530,6 +3533,9 @@ private void registerMbean(Object o, @Nullable String cacheName, boolean near)
      * @param near Near flag.
      */
     private void unregisterMbean(Object o, @Nullable String cacheName, boolean near) {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
+
         assert o != null;
 
         MBeanServer srvr = ctx.config().getMBeanServer();
@@ -3545,7 +3551,7 @@ private void unregisterMbean(Object o, @Nullable String cacheName, boolean near)
                 try {
                     srvr.unregisterMBean(U.makeCacheMBeanName(ctx.gridName(), cacheName, o.getClass().getName()));
                 }
-                catch (JMException e) {
+                catch (Throwable e) {
                     U.error(log, "Failed to unregister MBean for component: " + o, e);
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index c2efb953c9135..84566872e5abf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -134,6 +134,7 @@
 import java.util.zip.ZipOutputStream;
 import javax.management.DynamicMBean;
 import javax.management.JMException;
+import javax.management.MBeanRegistrationException;
 import javax.management.MBeanServer;
 import javax.management.MalformedObjectNameException;
 import javax.management.ObjectName;
@@ -150,10 +151,10 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteDeploymentException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteIllegalStateException;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryRawReader;
 import org.apache.ignite.binary.BinaryRawWriter;
 import org.apache.ignite.cluster.ClusterGroupEmptyException;
@@ -509,6 +510,9 @@ public abstract class IgniteUtils {
         }
     };
 
+    /** Ignite MBeans disabled flag. */
+    public static boolean IGNITE_MBEANS_DISABLED = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_MBEANS_DISABLED);
+
     /**
      * Initializes enterprise check.
      */
@@ -4423,10 +4427,13 @@ public static ObjectName makeCacheMBeanName(@Nullable String gridName, @Nullable
      * @param impl MBean implementation.
      * @param itf MBean interface.
      * @return JMX object name.
+     * @throws MBeanRegistrationException if MBeans are disabled.
      * @throws JMException If MBean creation failed.
      */
     public static <T> ObjectName registerMBean(MBeanServer mbeanSrv, @Nullable String gridName, @Nullable String grp,
         String name, T impl, @Nullable Class<T> itf) throws JMException {
+        if(IGNITE_MBEANS_DISABLED)
+            throw new MBeanRegistrationException(new IgniteIllegalStateException("No MBeans are allowed."));
         assert mbeanSrv != null;
         assert name != null;
         assert itf != null;
@@ -4447,10 +4454,14 @@ public static <T> ObjectName registerMBean(MBeanServer mbeanSrv, @Nullable Strin
      * @param impl MBean implementation.
      * @param itf MBean interface.
      * @return JMX object name.
+     * @throws MBeanRegistrationException if MBeans are disabled.
      * @throws JMException If MBean creation failed.
      */
     public static <T> ObjectName registerMBean(MBeanServer mbeanSrv, ObjectName name, T impl, Class<T> itf)
         throws JMException {
+        if(IGNITE_MBEANS_DISABLED)
+            throw new MBeanRegistrationException(new IgniteIllegalStateException("MBeans are disabled."));
+
         assert mbeanSrv != null;
         assert name != null;
         assert itf != null;
@@ -4473,10 +4484,14 @@ public static <T> ObjectName registerMBean(MBeanServer mbeanSrv, ObjectName name
      * @param impl MBean implementation.
      * @param itf MBean interface.
      * @return JMX object name.
+     * @throws MBeanRegistrationException if MBeans are disabled.
      * @throws JMException If MBean creation failed.
      */
     public static <T> ObjectName registerCacheMBean(MBeanServer mbeanSrv, @Nullable String gridName,
         @Nullable String cacheName, String name, T impl, Class<T> itf) throws JMException {
+        if(IGNITE_MBEANS_DISABLED)
+            throw new MBeanRegistrationException(new IgniteIllegalStateException("MBeans are disabled."));
+
         assert mbeanSrv != null;
         assert name != null;
         assert itf != null;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index 8879364dca5dd..e2350f0a50a3d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -410,6 +410,8 @@ private static String format(String msg, Object locVal, Object rmtVal) {
      */
     protected final <T extends IgniteSpiManagementMBean> void registerMBean(String gridName, T impl, Class<T> mbeanItf)
         throws IgniteSpiException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
         MBeanServer jmx = ignite.configuration().getMBeanServer();
 
         assert mbeanItf == null || mbeanItf.isInterface();
@@ -434,6 +436,7 @@ protected final <T extends IgniteSpiManagementMBean> void registerMBean(String g
     protected final void unregisterMBean() throws IgniteSpiException {
         // Unregister SPI MBean.
         if (spiMBean != null) {
+            assert !U.IGNITE_MBEANS_DISABLED;
             MBeanServer jmx = ignite.configuration().getMBeanServer();
 
             assert jmx != null;
diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java
new file mode 100644
index 0000000000000..f08f58bd9258a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util.mbeans;
+
+import java.util.concurrent.Callable;
+import javax.management.MBeanRegistrationException;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Disabling MBeans test.
+ */
+public class GridMBeanDisableSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        IgniteUtils.IGNITE_MBEANS_DISABLED = true;
+
+        super.beforeTestsStarted();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        IgniteUtils.IGNITE_MBEANS_DISABLED = false;
+    }
+
+    /**
+     * Test MBean registration.
+     *
+     * @throws Exception Thrown if test fails.
+     */
+    public void testCorrectMBeanInfo() throws Exception {
+        // Node should start and stopped with no errors.
+        try (final Ignite ignite = startGrid(0)) {
+
+            // Cache should be created and closed with no errors.
+            try (IgniteCache cache = ignite.getOrCreateCache(new CacheConfiguration("MyCache"))) {
+
+                final MBeanServer server = ignite.configuration().getMBeanServer();
+
+                GridTestUtils.assertThrowsWithCause(
+                    new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            U.registerMBean(server, ignite.name(), "dummy", "DummyMbean1", new DummyMBeanImpl(), DummyMBean.class);
+
+                            return null;
+
+                        }
+                    }, MBeanRegistrationException.class);
+
+                GridTestUtils.assertThrowsWithCause(
+                    new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            ObjectName objName = U.makeMBeanName(
+                                ignite.name(),
+                                "dummy",
+                                "DummyMbean2"
+                            );
+
+                            U.registerMBean(server, objName, new DummyMBeanImpl(), DummyMBean.class);
+
+                            return null;
+
+                        }
+                    }, MBeanRegistrationException.class);
+
+                GridTestUtils.assertThrowsWithCause(
+                    new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            U.registerCacheMBean(server, ignite.name(), "MyCache", "DummyMbean3",
+                                new DummyMBeanImpl(), DummyMBean.class);
+
+                            return null;
+
+                        }
+                    }, MBeanRegistrationException.class);
+            }
+        }
+    }
+
+    /**
+     * MBean dummy interface.
+     */
+    interface DummyMBean {
+        /** */
+        void noop();
+    }
+
+    /**
+     * MBean stub.
+     */
+    static class DummyMBeanImpl implements DummyMBean {
+        /** {@inheritDoc} */
+        @Override public void noop() {
+            // No op.
+        }
+    }
+}
\ No newline at end of file

From e4d141e97ab4ec34b5fe6a7bc599413223944438 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 14 Jul 2017 14:40:02 +0300
Subject: [PATCH 291/516] IGNITE-5103 - Server drops client node from cluster
 when no heartbeat messages received in interval heartBeatsFrequency *
 maxMissedClientHeartBeats.

---
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  46 +++++-
 ...entDiscoverySpiFailureTimeoutSelfTest.java |  14 ++
 .../tcp/TcpClientDiscoverySpiSelfTest.java    |   2 +-
 .../TcpDiscoveryClientSuspensionSelfTest.java | 133 ++++++++++++++++++
 4 files changed, 193 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 6e5af2986dcff..75d8f83344d25 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6386,6 +6386,12 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
         /** Current client metrics. */
         private volatile ClusterMetrics metrics;
 
+        /** Last heartbeat message receive time. */
+        private volatile long lastHbMsgTime;
+
+        /** Max heartbeat message receive interval. */
+        private final long maxHbInterval;
+
         /** */
         private final AtomicReference<GridFutureAdapter<Boolean>> pingFut = new AtomicReference<>();
 
@@ -6397,10 +6403,13 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
          * @param clientNodeId Node ID.
          */
         protected ClientMessageWorker(Socket sock, UUID clientNodeId) throws IOException {
-            super("tcp-disco-client-message-worker", 2000);
+            super("tcp-disco-client-message-worker", Math.max(spi.hbFreq, 10));
 
             this.sock = sock;
             this.clientNodeId = clientNodeId;
+
+            maxHbInterval = spi.hbFreq * spi.maxMissedClientHbs;
+            lastHbMsgTime = U.currentTimeMillis();
         }
 
         /**
@@ -6421,6 +6430,9 @@ ClusterMetrics metrics() {
          * @param metrics New current client metrics.
          */
         void metrics(ClusterMetrics metrics) {
+            // Updates only on heartbeat message.
+            lastHbMsgTime = U.currentTimeMillis();
+
             this.metrics = metrics;
         }
 
@@ -6597,6 +6609,38 @@ public boolean ping(IgniteSpiOperationTimeoutHelper timeoutHelper) throws Interr
             }
         }
 
+        /** {@inheritDoc} */
+        @Override protected void noMessageLoop() {
+            if (U.currentTimeMillis() - lastHbMsgTime > maxHbInterval) {
+                TcpDiscoveryNode clientNode = ring.node(clientNodeId);
+
+                if (clientNode != null) {
+                    boolean failedNode;
+
+                    synchronized (mux) {
+                        failedNode = failedNodes.containsKey(clientNode);
+                    }
+
+                    if (!failedNode) {
+                        String msg = "Client node considered as unreachable " +
+                            "and will be dropped from cluster, " +
+                            "because no heartbeat messages received in interval: " +
+                            "TcpDiscoverySpi.getHeartbeatFrequency() * TcpDiscoverySpi.getMaxMissedClientHeartbeats() ms. " +
+                            "It maybe caused by network problems or long GC pause on client node, try to increase mentioned " +
+                            "parameters. " +
+                            "[nodeId=" + clientNodeId +
+                            ", heartBeatFrequency=" + spi.hbFreq +
+                            ", maxMissedClientHeartbeats=" + spi.maxMissedClientHbs +
+                            ']';
+
+                        failNode(clientNodeId, msg);
+
+                        U.warn(log, msg);
+                    }
+                }
+            }
+        }
+
         /** {@inheritDoc} */
         @Override protected void cleanup() {
             super.cleanup();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiFailureTimeoutSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiFailureTimeoutSelfTest.java
index 35aa934e912a2..cc26c1cda6288 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiFailureTimeoutSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiFailureTimeoutSelfTest.java
@@ -185,6 +185,20 @@ public void testClientReconnectOnCoordinatorRouterFail2() throws Exception {
         clientReconnectOnCoordinatorRouterFail(2);
     }
 
+    /** {@inheritDoc} */
+    @Override public void testPingFailedClientNode() throws Exception {
+        int hb = maxMissedClientHbs;
+
+        maxMissedClientHbs = Integer.MAX_VALUE;
+
+        try {
+            super.testPingFailedClientNode();
+        }
+        finally {
+            maxMissedClientHbs = hb;
+        }
+    }
+
     /**
      * Test tries to provoke scenario when client sends reconnect message before router failure detected.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 0483a1ce8fbc6..419497753bbc2 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -142,7 +142,7 @@ public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     private boolean longSockTimeouts;
 
     /** */
-    private int maxMissedClientHbs = TcpDiscoverySpi.DFLT_MAX_MISSED_CLIENT_HEARTBEATS;
+    protected int maxMissedClientHbs = TcpDiscoverySpi.DFLT_MAX_MISSED_CLIENT_HEARTBEATS;
 
     /** */
     private IgniteInClosure2X<TcpDiscoveryAbstractMessage, Socket> afterWrite;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
new file mode 100644
index 0000000000000..7a1dd1ba23e5c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.Timer;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for missed client heartbeats.
+ */
+public class TcpDiscoveryClientSuspensionSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+        disco.setHeartbeatFrequency(200);
+        disco.setMaxMissedClientHeartbeats(10);
+
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY, "10000");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneServer() throws Exception {
+        doTestClientSuspension(1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTwoServers() throws Exception {
+        doTestClientSuspension(2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testThreeServers() throws Exception {
+        doTestClientSuspension(3);
+    }
+
+    /**
+     * @param serverCnt Servers count.
+     * @throws Exception If failed.
+     */
+    private void doTestClientSuspension(int serverCnt) throws Exception {
+        startGrids(serverCnt);
+
+        Ignition.setClientMode(true);
+
+        Ignite client = startGrid("client");
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(1, grid(i).cluster().forClients().nodes().size());
+
+        Thread.sleep(3000);
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(1, grid(i).cluster().forClients().nodes().size());
+
+        suspendClientHeartbeats(client);
+
+        Thread.sleep(3000);
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(0, grid(i).cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @param client Client.
+     */
+    private void suspendClientHeartbeats(Ignite client) {
+        assert client.cluster().localNode().isClient();
+
+        ClientImpl impl = U.field(client.configuration().getDiscoverySpi(), "impl");
+
+        Timer timer = U.field(impl, "timer");
+
+        timer.cancel();
+
+        System.out.println("Heartbeats suspended");
+    }
+}

From 45573945066113fd29548699f23c2bc9f22cef36 Mon Sep 17 00:00:00 2001
From: Tikhonov Nikolay <tikhonovnicolay@gmail.com>
Date: Wed, 21 Jun 2017 17:55:05 +0300
Subject: [PATCH 292/516] ignite-5489 Fixed possible connection leaks when
 loadPreviousValue set to true

---
 .../store/GridCacheStoreManagerAdapter.java   |   9 +-
 .../cache/CacheConnectionLeakStoreTxTest.java | 287 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   2 +
 3 files changed, 295 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 11d9816ff1e6d..c35b8fb9d2c01 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -338,7 +338,12 @@ private CacheStore cacheStoreWrapper(GridKernalContext ctx,
                 throw new IgniteCheckedException(new CacheLoaderException(e));
             }
             finally {
-                sessionEnd0(tx, threwEx);
+                IgniteInternalTx tx0 = tx;
+
+                if (tx0 != null && (tx0.dht() && tx0.local()))
+                    tx0 = null;
+
+                sessionEnd0(tx0, threwEx);
             }
 
             if (log.isDebugEnabled())
@@ -867,8 +872,6 @@ private void sessionEnd0(@Nullable IgniteInternalTx tx, boolean threwEx) throws
                         lsnr.onSessionEnd(locSes, !threwEx);
                 }
 
-                assert !sesHolder.get().ended(store);
-
                 store.sessionEnd(!threwEx);
             }
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java
new file mode 100644
index 0000000000000..f6b735077e1d8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.Serializable;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.cache.store.CacheStoreSession;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.resources.CacheStoreSessionResource;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.cache.TestCacheSession;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ *
+ */
+public class CacheConnectionLeakStoreTxTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int CLIENT_NODE = 1;
+
+    /** */
+    private static boolean client;
+
+    /** */
+    private static volatile boolean isLoadFromStore;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(disco);
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrid(0);
+
+        client = true;
+
+        startGrid(CLIENT_NODE);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        isLoadFromStore = false;
+        TestStore.sessions.clear();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupAtomic() throws Exception {
+        checkConnectionLeak(CacheAtomicityMode.ATOMIC, null, null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupAtomicLoadFromStore() throws Exception {
+        isLoadFromStore = true;
+
+        checkConnectionLeak(CacheAtomicityMode.ATOMIC, null, null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupOptimisticRepeatableRead() throws Exception {
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, OPTIMISTIC, REPEATABLE_READ);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupOptimisticRepeatableReadLoadFromStore() throws Exception {
+        isLoadFromStore = true;
+
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, OPTIMISTIC, REPEATABLE_READ);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupOptimisticReadCommitted() throws Exception {
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, OPTIMISTIC, READ_COMMITTED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupOptimisticReadCommittedLoadFromStore() throws Exception {
+        isLoadFromStore = true;
+
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, OPTIMISTIC, READ_COMMITTED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupPessimisticRepeatableRead() throws Exception {
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, PESSIMISTIC, REPEATABLE_READ);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupPessimisticReadCommitted() throws Exception {
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, PESSIMISTIC, READ_COMMITTED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLeakOneBackupPessimisticReadCommittedLoadFromStore() throws Exception {
+        isLoadFromStore = true;
+
+        checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, PESSIMISTIC, READ_COMMITTED);
+    }
+
+    /**
+     * @param atomicityMode Atomicity mode.
+     * @param txConcurrency Transaction concurrency.
+     * @param txIsolation Transaction isolation.
+     *
+     * @throws Exception If failed.
+     */
+    private void checkConnectionLeak(
+            CacheAtomicityMode atomicityMode,
+            TransactionConcurrency txConcurrency,
+            TransactionIsolation txIsolation
+    ) throws Exception {
+        CacheConfiguration<Integer, Integer> cacheCfg = new CacheConfiguration<>();
+
+        cacheCfg.setCacheMode(CacheMode.PARTITIONED);
+        cacheCfg.setAtomicityMode(atomicityMode);
+        cacheCfg.setCacheStoreFactory(new TestStoreFactory());
+        cacheCfg.setReadThrough(true);
+        cacheCfg.setWriteThrough(false);
+        cacheCfg.setLoadPreviousValue(true);
+
+        Ignite ignite = ignite(CLIENT_NODE);
+        IgniteCache<Integer, Integer> cache = ignite.createCache(cacheCfg);
+
+        try {
+            assertEquals(0, cache.size());
+
+            if (atomicityMode == CacheAtomicityMode.TRANSACTIONAL) {
+                try (Transaction tx = ignite.transactions().txStart(txConcurrency, txIsolation)) {
+                    cacheOp(cache);
+
+                    tx.commit();
+                }
+            }
+            else {
+                cacheOp(cache);
+            }
+
+            assertTrue("Session was leak on nodes: " + TestStore.sessions, TestStore.sessions.isEmpty());
+        }
+        finally {
+            cache.destroy();
+        }
+    }
+
+    /**
+     * @param cache Cache.
+     */
+    private void cacheOp(IgniteCache<Integer, Integer> cache) {
+        boolean b = cache.putIfAbsent(42, 42);
+
+        log.info("PutIfAbsent: " + b);
+
+        Integer val = cache.get(42);
+
+        log.info("Get: " + val);
+    }
+
+    /**
+     *
+     */
+    private static class TestStoreFactory implements Factory<CacheStoreAdapter<Integer, Integer>> {
+        /** {@inheritDoc} */
+        @Override public CacheStoreAdapter<Integer, Integer> create() {
+            return new TestStore();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestStore extends CacheStoreAdapter<Integer, Integer> implements Serializable {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** */
+        @CacheStoreSessionResource
+        private CacheStoreSession ses;
+
+        /** */
+        private CacheStoreSession NULL = new TestCacheSession();
+
+        /** */
+        public static ConcurrentHashMap<CacheStoreSession, ClusterNode> sessions = new ConcurrentHashMap<>();
+
+        /** {@inheritDoc} */
+        @Override public Integer load(Integer key) throws CacheLoaderException {
+            addSession();
+
+            return isLoadFromStore ? key : null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Integer, ? extends Integer> e) throws CacheWriterException {
+            addSession();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            addSession();
+        }
+
+        /**  */
+        private void addSession() {
+            sessions.put(ses == null ? NULL : ses, ignite.cluster().localNode());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sessionEnd(boolean commit) {
+            sessions.remove(ses == null ? NULL : ses);
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 2b446bbea8e6d..c1905af9db331 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.cache.store.jdbc.CacheJdbcStoreSessionListenerSelfTest;
 import org.apache.ignite.internal.processors.GridCacheTxLoadFromStoreOnLockSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheClientStoreSelfTest;
+import org.apache.ignite.internal.processors.cache.CacheConnectionLeakStoreTxTest;
 import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticReadCommittedSeltTest;
 import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticRepeatableReadSeltTest;
 import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticSerializableSeltTest;
@@ -279,6 +280,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheStoreUsageMultinodeStaticStartTxTest.class);
         suite.addTestSuite(CacheStoreUsageMultinodeDynamicStartAtomicTest.class);
         suite.addTestSuite(CacheStoreUsageMultinodeDynamicStartTxTest.class);
+        suite.addTestSuite(CacheConnectionLeakStoreTxTest.class);
 
         suite.addTestSuite(GridCacheStoreManagerDeserializationTest.class);
         suite.addTestSuite(GridLocalCacheStoreManagerDeserializationTest.class);

From 37535634ef3325aaf9923fd17d24038dfd5cee38 Mon Sep 17 00:00:00 2001
From: agura <agura@gridgain.com>
Date: Tue, 11 Jul 2017 16:24:54 +0300
Subject: [PATCH 293/516] ignite-5722 Cache entries stay in onheap after scan
 query execution for OFFHEAP_TIRED cache with expiry policy

---
 .../cache/query/GridCacheQueryManager.java    |   7 +-
 .../ScanQueryOffheapExpiryPolicySelfTest.java | 112 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite2.java       |   3 +
 3 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/ScanQueryOffheapExpiryPolicySelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 14b1106eb5e1c..7efb746d4428e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -3678,7 +3678,12 @@ private CacheObject value(GridCacheEntryEx entry, KeyCacheObject key) throws Ign
                     if (expiryPlc != null)
                         entry.unswap();
 
-                    return entry.peek(true, true, true, topVer, expiryPlc);
+                    CacheObject cacheObj = entry.peek(true, true, true, topVer, expiryPlc);
+
+                    if (expiryPlc != null)
+                        cctx.evicts().touch(entry, topVer);
+
+                    return cacheObj;
                 }
                 catch (GridCacheEntryRemovedException ignore) {
                     entry = null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/ScanQueryOffheapExpiryPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/ScanQueryOffheapExpiryPolicySelfTest.java
new file mode 100644
index 0000000000000..e59e458b2371c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/ScanQueryOffheapExpiryPolicySelfTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import javax.cache.expiry.CreatedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.ignite.cache.CachePeekMode.OFFHEAP;
+import static org.apache.ignite.cache.CachePeekMode.ONHEAP;
+
+/**
+ *
+ */
+public class ScanQueryOffheapExpiryPolicySelfTest extends GridCommonAbstractTest {
+
+    /** Nodes count. */
+    private static final int NODES_CNT = 2;
+
+    /** Entries count */
+    private static final int ENTRIES_CNT = 1024;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = defaultCacheConfiguration();
+
+        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+        ccfg.setCacheMode(CacheMode.PARTITIONED);
+        ccfg.setBackups(1);
+        ccfg.setMemoryMode(CacheMemoryMode.OFFHEAP_TIERED);
+        ccfg.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.MINUTES, 10)));
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGridsMultiThreaded(NODES_CNT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testEntriesMovedFromOnHeap() throws Exception {
+        Ignite ignite0 = grid(0);
+        Ignite ignite1 = grid(1);
+
+        IgniteCache<Integer, Integer> cache0 = ignite0.cache(null);
+        IgniteCache<Integer, Integer> cache1 = ignite1.cache(null);
+
+        populateCache(cache0);
+
+        assertEquals(0, cache0.localSize(ONHEAP));
+        assertEquals(0, cache1.localSize(ONHEAP));
+
+        assertEquals(ENTRIES_CNT, cache0.localSize(OFFHEAP) + cache1.localSize(OFFHEAP));
+
+        cache0.query(new ScanQuery<>()).getAll();
+        cache1.query(new ScanQuery<>()).getAll();
+
+        assertEquals(0, cache0.localSize(ONHEAP));
+        assertEquals(0, cache1.localSize(ONHEAP));
+
+        assertEquals(ENTRIES_CNT, cache0.localSize(OFFHEAP) + cache1.localSize(OFFHEAP));
+    }
+
+    /**
+     * @param cache Cache instance.
+     */
+    private static void populateCache(IgniteCache<Integer, Integer> cache) {
+        for (int i = 0; i < ENTRIES_CNT; i++)
+            cache.put(i, i);
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index 8ac219f89f199..0241f86198420 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -50,6 +50,7 @@
 import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalQueryCancelOrTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.query.GridCacheSwapScanQuerySelfTest;
+import org.apache.ignite.internal.processors.cache.query.ScanQueryOffheapExpiryPolicySelfTest;
 import org.apache.ignite.internal.processors.query.h2.sql.BaseH2CompareQueryTest;
 import org.apache.ignite.internal.processors.query.h2.sql.H2CompareBigQueryTest;
 import org.apache.ignite.spi.communication.tcp.GridOrderedMessageCancelSelfTest;
@@ -88,6 +89,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(CacheQueryOffheapEvictDataLostTest.class);
 
+        suite.addTestSuite(ScanQueryOffheapExpiryPolicySelfTest.class);
+
         // Ignite cache and H2 comparison.
         suite.addTestSuite(BaseH2CompareQueryTest.class);
         suite.addTestSuite(H2CompareBigQueryTest.class);

From c3e2eebeccbdc4bb3a7a0a70d09a8a7b63399c2c Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Tue, 18 Jul 2017 18:50:48 +0300
Subject: [PATCH 294/516] IGNITE 5776 Add option to turn on filter reachable
 addresses in TcpCommunicationSpi

---
 .../tcp/TcpCommunicationSpi.java              | 77 ++++++++++++++-----
 1 file changed, 56 insertions(+), 21 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 0d80f4778df36..7a5f7c1f16f3a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -313,6 +313,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     /** Default value for {@code TCP_NODELAY} socket option (value is <tt>true</tt>). */
     public static final boolean DFLT_TCP_NODELAY = true;
 
+    /** Default value for {@code FILTER_REACHABLE_ADDRESSES} socket option (value is <tt>false</tt>). */
+    public static final boolean DFLT_FILTER_REACHABLE_ADDRESSES = false;
+
     /** Default received messages threshold for sending ack. */
     public static final int DFLT_ACK_SND_THRESHOLD = 32;
 
@@ -996,6 +999,9 @@ class ConnectClosure implements IgniteInClosure<Boolean> {
     /** {@code TCP_NODELAY} option value for created sockets. */
     private boolean tcpNoDelay = DFLT_TCP_NODELAY;
 
+    /** {@code FILTER_REACHABLE_ADDRESSES} option value for created sockets. */
+    private boolean filterReachableAddresses = DFLT_FILTER_REACHABLE_ADDRESSES;
+
     /** Number of received messages after which acknowledgment is sent. */
     private int ackSndThreshold = DFLT_ACK_SND_THRESHOLD;
 
@@ -1515,6 +1521,33 @@ public void setTcpNoDelay(boolean tcpNoDelay) {
         return tcpNoDelay;
     }
 
+    /**
+     * Gets value for {@code FILTER_REACHABLE_ADDRESSES} socket option.
+     *
+     * @return {@code True} if needed to filter reachable addresses.
+     */
+    public boolean isFilterReachableAddresses() {
+        return filterReachableAddresses;
+    }
+
+    /**
+     * Setting this option to {@code true} enables filter for reachable
+     * addresses on creating tcp client.
+     * <p>
+     * Usually its advised to set this value to {@code false}.
+     * <p>
+     * If not provided, default value is {@link #DFLT_FILTER_REACHABLE_ADDRESSES}.
+     *
+     * @param filterReachableAddresses {@code True} to filter reachable addresses.
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public TcpCommunicationSpi setFilterReachableAddresses(boolean filterReachableAddresses) {
+        this.filterReachableAddresses = filterReachableAddresses;
+
+        return this;
+    }
+
     /**
      * Sets receive buffer size for sockets created or accepted by this SPI.
      * <p>
@@ -2780,35 +2813,37 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
         if (isExtAddrsExist)
             addrs.addAll(extAddrs);
 
-        Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
+        if (filterReachableAddresses) {
+            Set<InetAddress> allInetAddrs = U.newHashSet(addrs.size());
 
-        for (InetSocketAddress addr : addrs) {
-            // Skip unresolved as addr.getAddress() can return null.
-            if(!addr.isUnresolved())
-                allInetAddrs.add(addr.getAddress());
-        }
+            for (InetSocketAddress addr : addrs) {
+                // Skip unresolved as addr.getAddress() can return null.
+                if (!addr.isUnresolved())
+                    allInetAddrs.add(addr.getAddress());
+            }
 
-        List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
+            List<InetAddress> reachableInetAddrs = U.filterReachable(allInetAddrs);
 
-        if (reachableInetAddrs.size() < allInetAddrs.size()) {
-            LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
+            if (reachableInetAddrs.size() < allInetAddrs.size()) {
+                LinkedHashSet<InetSocketAddress> addrs0 = U.newLinkedHashSet(addrs.size());
 
-            List<InetSocketAddress> unreachableInetAddr = new ArrayList<>(allInetAddrs.size() - reachableInetAddrs.size());
+                List<InetSocketAddress> unreachableInetAddr = new ArrayList<>(allInetAddrs.size() - reachableInetAddrs.size());
 
-            for (InetSocketAddress addr : addrs) {
-                if (reachableInetAddrs.contains(addr.getAddress()))
-                    addrs0.add(addr);
-                else
-                    unreachableInetAddr.add(addr);
-            }
+                for (InetSocketAddress addr : addrs) {
+                    if (reachableInetAddrs.contains(addr.getAddress()))
+                        addrs0.add(addr);
+                    else
+                        unreachableInetAddr.add(addr);
+                }
 
-            addrs0.addAll(unreachableInetAddr);
+                addrs0.addAll(unreachableInetAddr);
 
-            addrs = addrs0;
-        }
+                addrs = addrs0;
+            }
 
-        if (log.isDebugEnabled())
-            log.debug("Addresses to connect for node [rmtNode=" + node.id() + ", addrs=" + addrs.toString() + ']');
+            if (log.isDebugEnabled())
+                log.debug("Addresses to connect for node [rmtNode=" + node.id() + ", addrs=" + addrs.toString() + ']');
+        }
 
         boolean conn = false;
         GridCommunicationClient client = null;

From 97d3f42c1c95a6aafce1d0c300ccfe6708398c17 Mon Sep 17 00:00:00 2001
From: shtykh_roman <rshtykh@yahoo.com>
Date: Wed, 7 Sep 2016 08:35:31 +0300
Subject: [PATCH 295/516] IGNITE-3809: Fix for ArrayIndexOutOfBoundsException
 in GridUnsafeLru.

(cherry picked from commit 31b9bb8)
---
 .../util/offheap/unsafe/GridUnsafeLru.java    | 30 ++++++++++++++++---
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java
index aaff4f93a94f7..ea652171d8b2c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java
@@ -28,8 +28,7 @@
 /**
  * Striped LRU queue.
  */
-@SuppressWarnings("ForLoopReplaceableByForEach")
-class GridUnsafeLru {
+@SuppressWarnings("ForLoopReplaceableByForEach") class GridUnsafeLru {
     /** Number of stripes. */
     private final short cnt;
 
@@ -47,6 +46,9 @@ class GridUnsafeLru {
     /** Current round-robin remove stripe index. */
     private final AtomicInteger rmvIdx;
 
+    /** Max stripe index count. */
+    private final int maxIdxCnt;
+
     /** Released flag. */
     private AtomicBoolean released = new AtomicBoolean(false);
 
@@ -68,6 +70,8 @@ class GridUnsafeLru {
 
         addIdx = new AtomicInteger();
         rmvIdx = new AtomicInteger(cnt / 2);
+
+        maxIdxCnt = cnt - 1;
     }
 
     /**
@@ -156,7 +160,7 @@ long entry(short order, long qAddr) {
      * @throws GridOffHeapOutOfMemoryException If failed.
      */
     long offer(int part, long addr, int hash) throws GridOffHeapOutOfMemoryException {
-        return lrus[addIdx.getAndIncrement() % cnt].offer(part, addr, hash);
+        return lrus[incrementAndGet(addIdx, maxIdxCnt)].offer(part, addr, hash);
     }
 
     /**
@@ -165,7 +169,7 @@ long offer(int part, long addr, int hash) throws GridOffHeapOutOfMemoryException
      * @return Queue node address.
      */
     long prePoll() {
-        int idx = rmvIdx.getAndIncrement();
+        int idx = incrementAndGet(rmvIdx, maxIdxCnt);
 
         // Must try to poll from each LRU.
         for (int i = 0; i < lrus.length; i++) {
@@ -180,6 +184,7 @@ long prePoll() {
 
     /**
      * Removes polling node from the queue.
+     *
      * @param qAddr Queue node address.
      */
     void poll(long qAddr) {
@@ -215,6 +220,23 @@ void destruct() {
         }
     }
 
+    /**
+     * Atomically increments the given value by one, re-starting from 0 when the specified maximum is reached.
+     *
+     * @param value Value to increment.
+     * @param max Maximum after reaching which the value is reset to 0.
+     * @return Incremented value.
+     */
+    private int incrementAndGet(AtomicInteger value, int max) {
+        while (true) {
+            int cur = value.get();
+            int next = cur == max ? 0 : cur + 1;
+
+            if (value.compareAndSet(cur, next))
+                return next;
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridUnsafeLru.class, this);

From c2062d52a227dda5afee560d80c3bb4dd2ce09eb Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 19 Jul 2017 08:41:46 +0300
Subject: [PATCH 296/516] Remove empty test_utils.cpp

---
 modules/platforms/cpp/core-test/src/test_utils.cpp | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 modules/platforms/cpp/core-test/src/test_utils.cpp

diff --git a/modules/platforms/cpp/core-test/src/test_utils.cpp b/modules/platforms/cpp/core-test/src/test_utils.cpp
deleted file mode 100644
index e69de29bb2d1d..0000000000000

From 45cbba4853bab1ba4ffe2ea0d3add99a9d454aab Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 19 Jul 2017 10:44:04 +0300
Subject: [PATCH 297/516] IGNITE-5768 - Retry resolving class name from
 marshaller cache and .classname file.

---
 .../apache/ignite/IgniteSystemProperties.java |  3 +
 .../internal/MarshallerContextImpl.java       | 61 ++++++++++++++-----
 2 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index aaba91e15d304..d3c9a6b64c650 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -580,6 +580,9 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_ENABLE_FORCIBLE_NODE_KILL = "IGNITE_ENABLE_FORCIBLE_NODE_KILL";
 
+    /** Ignite marshaller cache reread pause. */
+    public static final String IGNITE_MARSHALLER_CACHE_REREAD_PAUSE = "IGNITE_MARSHALLER_CACHE_REREAD_PAUSE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java
index f3e368d859a04..dae0d4edad1d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java
@@ -38,6 +38,7 @@
 import javax.cache.event.CacheEntryUpdatedListener;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.internal.processors.cache.CachePartialUpdateCheckedException;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheTryPutFailedException;
@@ -47,10 +48,18 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.plugin.PluginProvider;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_MARSHALLER_CACHE_REREAD_PAUSE;
+
 /**
  * Marshaller context implementation.
  */
 public class MarshallerContextImpl extends MarshallerContextAdapter {
+    /** Cache reread tries. */
+    private static final int CACHE_REREAD_TRIES = 5;
+
+    /** Cache reread pause. */
+    private static final int CACHE_REREAD_PAUSE = IgniteSystemProperties.getInteger(IGNITE_MARSHALLER_CACHE_REREAD_PAUSE, 20);
+
     /** */
     private static final GridStripedLock fileLock = new GridStripedLock(32);
 
@@ -193,35 +202,59 @@ public void onKernalStop() {
                 throw new IllegalStateException("Failed to initialize marshaller context (grid is stopping).");
         }
 
-        String clsName = cache0.getTopologySafe(id);
+        String clsName = null;
+
+        for (int i = 0; i < CACHE_REREAD_TRIES; i++) {
+            clsName = cache0.getTopologySafe(id);
+
+            if (clsName != null)
+                break;
+
+            U.sleep(CACHE_REREAD_PAUSE);
+        }
 
         if (clsName == null) {
             String fileName = id + ".classname";
 
-            Lock lock = fileLock(fileName);
+            // Class name may be not in the file yet.
+            for (int i = 0; i < 2; i++) {
+                Lock lock = fileLock(fileName);
+
+                lock.lock();
 
-            lock.lock();
+                File file;
 
-            try {
-                File file = new File(workDir, fileName);
+                try {
+                    file = new File(workDir, fileName);
 
-                try (FileInputStream in = new FileInputStream(file)) {
-                    FileLock fileLock = fileLock(in.getChannel(), true);
+                    try (FileInputStream in = new FileInputStream(file)) {
+                        FileLock fileLock = fileLock(in.getChannel(), true);
 
-                    assert fileLock != null : fileName;
+                        assert fileLock != null : fileName;
 
-                    try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
-                        clsName = reader.readLine();
+                        try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
+                            clsName = reader.readLine();
+                        }
+                    }
+                    catch (IOException ignore) {
+                        // Will fail on last try.
                     }
                 }
-                catch (IOException e) {
+                finally {
+                    lock.unlock();
+                }
+
+                if (clsName != null)
+                    break;
+
+                // Fail on second unsuccessful try.
+                if (i == 1) {
                     throw new IgniteCheckedException("Class definition was not found " +
                         "at marshaller cache and local file. " +
                         "[id=" + id + ", file=" + file.getAbsolutePath() + ']');
                 }
-            }
-            finally {
-                lock.unlock();
+
+                U.sleep(20);
             }
 
             // Must explicitly put entry to cache to invoke other continuous queries.

From f24969f7e908645444df622642967a5f7fd3db23 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Wed, 19 Jul 2017 19:30:07 +0300
Subject: [PATCH 298/516] IGNITE 5775 JobsProcessor fix bug with delay in
 compute

---
 .../processors/job/GridJobProcessor.java      | 10 +--
 .../IgniteComputeJobOneThreadTest.java        | 79 +++++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |  2 +
 3 files changed, 86 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeJobOneThreadTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index ea9cbd7dbd272..09f80847832a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -1780,6 +1780,11 @@ private class JobEventListener implements GridJobEventListener {
                         return;
                     }
 
+                    if (!activeJobs.remove(worker.getJobId(), worker))
+                        cancelledJobs.remove(worker.getJobId(), worker);
+
+                    heldJobs.remove(worker.getJobId());
+
                     try {
                         handleCollisions();
                     }
@@ -1787,11 +1792,6 @@ private class JobEventListener implements GridJobEventListener {
                         rwLock.readUnlock();
                     }
                 }
-
-                if (!activeJobs.remove(worker.getJobId(), worker))
-                    cancelledJobs.remove(worker.getJobId(), worker);
-
-                heldJobs.remove(worker.getJobId());
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeJobOneThreadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeJobOneThreadTest.java
new file mode 100644
index 0000000000000..ccd8fa3765f14
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeJobOneThreadTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.spi.collision.fifoqueue.FifoQueueCollisionSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test of absence of gaps between jobs in compute
+ */
+public class IgniteComputeJobOneThreadTest extends GridCommonAbstractTest {
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        FifoQueueCollisionSpi colSpi = new FifoQueueCollisionSpi();
+        colSpi.setParallelJobsNumber(1);
+
+        return super.getConfiguration(name)
+            .setMetricsUpdateFrequency(10000)
+            .setCollisionSpi(colSpi);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        startGrid(0);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 10000;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoTimeout() throws Exception {
+        Ignite ignite = ignite(0);
+
+        IgniteFuture fut = null;
+
+        IgniteCompute compute = ignite.compute().withAsync();
+
+        for (int i = 0; i < 10000; i++) {
+            compute.run(new IgniteRunnable() {
+                @Override public void run() {
+
+                }
+            });
+        }
+
+        fut = compute.future();
+        fut.get();
+
+        assertTrue(true);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 8a501fdb801d6..c72bfe670b677 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -63,6 +63,7 @@
 import org.apache.ignite.internal.GridTaskResultCacheSelfTest;
 import org.apache.ignite.internal.GridTaskTimeoutSelfTest;
 import org.apache.ignite.internal.IgniteComputeEmptyClusterGroupTest;
+import org.apache.ignite.internal.IgniteComputeJobOneThreadTest;
 import org.apache.ignite.internal.IgniteComputeTopologyExceptionTest;
 import org.apache.ignite.internal.IgniteExecutorServiceTest;
 import org.apache.ignite.internal.IgniteExplicitImplicitDeploymentSelfTest;
@@ -152,6 +153,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TaskNodeRestartTest.class);
         suite.addTestSuite(IgniteRoundRobinErrorAfterClientReconnectTest.class);
         suite.addTestSuite(PublicThreadpoolStarvationTest.class);
+        suite.addTestSuite(IgniteComputeJobOneThreadTest.class);
 
         return suite;
     }

From f3adb9559b42698771b0b9b5116dd535446d2bef Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgain.com>
Date: Wed, 26 Jul 2017 15:40:19 +0700
Subject: [PATCH 299/516] IGNITE-5781 Visor throws ClassCastException if cache
 store implementation is other than CacheJdbcPojoStore.

---
 .../ignite/internal/visor/cache/VisorCacheTypeMetadata.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
index f17e5889663fc..a9ab8e061d3cb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
@@ -108,7 +108,7 @@ public static Collection<VisorCacheTypeMetadata> list(Collection<QueryEntity> qr
             }
 
         // Add JDBC types.
-        if (factory != null && factory instanceof CacheJdbcPojoStoreFactory) {
+        if (factory instanceof CacheJdbcPojoStoreFactory) {
              CacheJdbcPojoStoreFactory jdbcFactory = (CacheJdbcPojoStoreFactory) factory;
 
             JdbcType[] jdbcTypes = jdbcFactory.getTypes();

From a58688f6cc6c5b114dcdd1b2fde43b7e1e5e0732 Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgain.com>
Date: Wed, 26 Jul 2017 15:40:19 +0700
Subject: [PATCH 300/516] IGNITE-5781 Visor throws ClassCastException if cache
 store implementation is other than CacheJdbcPojoStore. (cherry picked from
 commit f3adb95)

---
 .../ignite/internal/visor/cache/VisorCacheTypeMetadata.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
index c87ad05d824e6..46d72c3e4e7ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
@@ -109,7 +109,7 @@ public static Collection<VisorCacheTypeMetadata> list(Collection<QueryEntity> qr
             }
 
         // Add JDBC types.
-        if (factory != null && factory instanceof CacheJdbcPojoStoreFactory) {
+        if (factory instanceof CacheJdbcPojoStoreFactory) {
              CacheJdbcPojoStoreFactory jdbcFactory = (CacheJdbcPojoStoreFactory) factory;
 
             JdbcType[] jdbcTypes = jdbcFactory.getTypes();

From 5b8963e38254fcac3bdd255a95adf9f733ce08f2 Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgain.com>
Date: Wed, 26 Jul 2017 15:40:19 +0700
Subject: [PATCH 301/516] IGNITE-5781 Visor throws ClassCastException if cache
 store implementation is other than CacheJdbcPojoStore. (cherry picked from
 commit f3adb95)

---
 .../ignite/internal/visor/cache/VisorCacheTypeMetadata.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
index c87ad05d824e6..46d72c3e4e7ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheTypeMetadata.java
@@ -109,7 +109,7 @@ public static Collection<VisorCacheTypeMetadata> list(Collection<QueryEntity> qr
             }
 
         // Add JDBC types.
-        if (factory != null && factory instanceof CacheJdbcPojoStoreFactory) {
+        if (factory instanceof CacheJdbcPojoStoreFactory) {
              CacheJdbcPojoStoreFactory jdbcFactory = (CacheJdbcPojoStoreFactory) factory;
 
             JdbcType[] jdbcTypes = jdbcFactory.getTypes();

From b7d1fb25ceba20b82631bb2e926a0ad52bf19e9d Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 20 Jul 2017 17:43:17 +0300
Subject: [PATCH 302/516] Do not process partition exchange messages in striped
 pool.

(cherry picked from commit 3a33706)
---
 .../dht/preloader/GridDhtPartitionsAbstractMessage.java      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java
index 6e69161cb6ff2..34521e5cd2714 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java
@@ -65,6 +65,11 @@ protected GridDhtPartitionsAbstractMessage() {
         this.lastVer = lastVer;
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return Integer.MIN_VALUE;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean addDeploymentInfo() {
         return false;

From c338bb9f5ac8f34dccbac1f7058765c5ce4549a4 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 20 Jul 2017 17:49:55 +0300
Subject: [PATCH 303/516] Removed unnecessary discoCache.updateAlives.

(cherry picked from commit 07a0698)
---
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java          | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index d3e3701936786..ba47ffd435e06 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -460,8 +460,6 @@ public void init() throws IgniteInterruptedCheckedException {
         assert !dummy && !forcePreload : this;
 
         try {
-            discoCache.updateAlives(cctx.discovery());
-
             srvNodes = new ArrayList<>(discoCache.serverNodes());
 
             remaining.addAll(F.nodeIds(F.view(srvNodes, F.remoteNodes(cctx.localNodeId()))));

From e7fbe8fba6bcd427bebf6ab6fa8746bc47b0d42f Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 31 Jul 2017 17:20:29 +0300
Subject: [PATCH 304/516] IGNITE-4800: Lucene query may fails with NPE.

---
 .../query/h2/opt/GridLuceneDirectory.java     |  44 ++-
 .../query/h2/opt/GridLuceneFile.java          |  78 +++-
 .../query/h2/opt/GridLuceneIndex.java         |   3 +-
 .../query/h2/opt/GridLuceneInputStream.java   |  27 +-
 .../query/h2/opt/GridLuceneOutputStream.java  |   4 +-
 .../cache/GridCacheFullTextQuerySelfTest.java | 367 ++++++++++++++++++
 ...CacheFullTextQueryNodeJoiningSelfTest.java |   4 +
 .../IgniteCacheQuerySelfTestSuite.java        |   6 +
 8 files changed, 497 insertions(+), 36 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java
index 480922cbb53b4..44887180528f4 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java
@@ -25,7 +25,9 @@
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
@@ -66,10 +68,7 @@ public GridLuceneDirectory(GridUnsafeMemory mem) {
         // and the code below is resilient to map changes during the array population.
         Set<String> fileNames = fileMap.keySet();
 
-        List<String> names = new ArrayList<>(fileNames.size());
-
-        for (String name : fileNames)
-            names.add(name);
+        List<String> names = new ArrayList<>(fileNames);
 
         return names.toArray(new String[names.size()]);
     }
@@ -115,21 +114,25 @@ public GridLuceneDirectory(GridUnsafeMemory mem) {
     @Override public void deleteFile(String name) throws IOException {
         ensureOpen();
 
-        doDeleteFile(name);
+        doDeleteFile(name, false);
     }
 
     /**
      * Deletes file.
      *
      * @param name File name.
+     * @param onClose If on close directory;
      * @throws IOException If failed.
      */
-    private void doDeleteFile(String name) throws IOException {
+    private void doDeleteFile(String name, boolean onClose) throws IOException {
         GridLuceneFile file = fileMap.remove(name);
 
         if (file != null) {
             file.delete();
 
+            // All files should be closed when Directory is closing.
+            assert !onClose || !file.hasRefs() : "Possible memory leak, resource is not closed: " + file.toString();
+
             sizeInBytes.addAndGet(-file.getSizeInBytes());
         }
         else
@@ -142,7 +145,10 @@ private void doDeleteFile(String name) throws IOException {
 
         GridLuceneFile file = newRAMFile();
 
-        GridLuceneFile existing = fileMap.remove(name);
+        // Lock for using in stream. Will be unlocked on stream closing.
+        file.lockRef();
+
+        GridLuceneFile existing = fileMap.put(name, file);
 
         if (existing != null) {
             sizeInBytes.addAndGet(-existing.getSizeInBytes());
@@ -150,8 +156,6 @@ private void doDeleteFile(String name) throws IOException {
             existing.delete();
         }
 
-        fileMap.put(name, file);
-
         return new GridLuceneOutputStream(file);
     }
 
@@ -174,6 +178,16 @@ protected GridLuceneFile newRAMFile() {
         if (file == null)
             throw new FileNotFoundException(name);
 
+        // Lock for using in stream. Will be unlocked on stream closing.
+        file.lockRef();
+
+        if (!fileMap.containsKey(name)) {
+            // Unblock for deferred delete.
+            file.releaseRef();
+
+            throw new FileNotFoundException(name);
+        }
+
         return new GridLuceneInputStream(name, file);
     }
 
@@ -181,16 +195,24 @@ protected GridLuceneFile newRAMFile() {
     @Override public void close() {
         isOpen = false;
 
+        IgniteException errs = null;
+
         for (String fileName : fileMap.keySet()) {
             try {
-                doDeleteFile(fileName);
+                doDeleteFile(fileName, true);
             }
             catch (IOException e) {
-                throw new IllegalStateException(e);
+                if (errs == null)
+                    errs = new IgniteException("Error closing index directory.");
+
+                errs.addSuppressed(e);
             }
         }
 
         assert fileMap.isEmpty();
+
+        if (errs != null && !F.isEmpty(errs.getSuppressed()))
+            throw errs;
     }
 
     /**
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneFile.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneFile.java
index 3985f0987b74f..4191135fc17db 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneFile.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneFile.java
@@ -17,22 +17,16 @@
 
 package org.apache.ignite.internal.processors.query.h2.opt;
 
-import java.io.Serializable;
 import java.util.Arrays;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static org.apache.ignite.internal.processors.query.h2.opt.GridLuceneOutputStream.BUFFER_SIZE;
 
 /**
  * Lucene file.
  */
-public class GridLuceneFile implements Serializable {
-    /** */
-    private static final long serialVersionUID = 0L;
-
-    /** */
-    public static final AtomicInteger filesCnt = new AtomicInteger();
-
+public class GridLuceneFile {
     /** */
     private LongArray buffers = new LongArray();
 
@@ -45,6 +39,12 @@ public class GridLuceneFile implements Serializable {
     /** */
     private volatile long sizeInBytes;
 
+    /** */
+    private final AtomicLong refCnt = new AtomicLong();
+
+    /** */
+    private final AtomicBoolean deleted = new AtomicBoolean();
+
     /**
      * File used as buffer, in no RAMDirectory
      *
@@ -52,8 +52,6 @@ public class GridLuceneFile implements Serializable {
      */
     GridLuceneFile(GridLuceneDirectory dir) {
         this.dir = dir;
-
-        filesCnt.incrementAndGet();
     }
 
     /**
@@ -92,52 +90,90 @@ final long addBuffer() {
         return buf;
     }
 
+    /**
+     * Increment ref counter.
+     */
+    void lockRef() {
+        refCnt.incrementAndGet();
+    }
+
+    /**
+     * Decrement ref counter.
+     */
+    void releaseRef() {
+        refCnt.decrementAndGet();
+
+        deferredDelete();
+    }
+
+    /**
+     * Checks if there is file stream opened.
+     *
+     * @return {@code True} if file has external references.
+     */
+    boolean hasRefs() {
+        long refs = refCnt.get();
+
+        assert refs >= 0;
+
+        return refs != 0;
+    }
+
     /**
      * Gets address of buffer.
      *
      * @param idx Index.
      * @return Pointer.
      */
-    protected final synchronized long getBuffer(int idx) {
+    final synchronized long getBuffer(int idx) {
         return buffers.get(idx);
     }
 
     /**
      * @return Number of buffers.
      */
-    protected final synchronized int numBuffers() {
+    final synchronized int numBuffers() {
         return buffers.size();
     }
 
     /**
-     * Expert: allocate a new buffer.
-     * Subclasses can allocate differently.
+     * Expert: allocate a new buffer. Subclasses can allocate differently.
      *
      * @return allocated buffer.
      */
-    protected long newBuffer() {
+    private long newBuffer() {
         return dir.memory().allocate(BUFFER_SIZE);
     }
 
     /**
      * Deletes file and deallocates memory..
      */
-    public synchronized void delete() {
-        if (buffers == null)
+    public void delete() {
+        if (!deleted.compareAndSet(false, true))
             return;
 
+        deferredDelete();
+    }
+
+    /**
+     * Deferred delete.
+     */
+    synchronized void deferredDelete() {
+        if (!deleted.get() || hasRefs())
+            return;
+
+        assert refCnt.get() == 0;
+
         for (int i = 0; i < buffers.idx; i++)
             dir.memory().release(buffers.arr[i], BUFFER_SIZE);
 
         buffers = null;
-
-        filesCnt.decrementAndGet();
     }
 
     /**
      * @return Size in bytes.
      */
-    public long getSizeInBytes() {
+    long getSizeInBytes() {
         return sizeInBytes;
     }
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java
index 716c9cb2c2af4..2242f1447d8f9 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java
@@ -290,7 +290,8 @@ public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> query(String qry,
     /** {@inheritDoc} */
     @Override public void close() {
         U.closeQuiet(writer);
-        U.closeQuiet(dir);
+
+        dir.close();
     }
 
     /**
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java
index eda97f36f487b..7740a03f9eb54 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java
@@ -20,6 +20,7 @@
 import java.io.EOFException;
 import java.io.IOException;
 import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 
@@ -28,7 +29,7 @@
 /**
  * A memory-resident {@link IndexInput} implementation.
  */
-public class GridLuceneInputStream extends IndexInput {
+public class GridLuceneInputStream extends IndexInput implements Cloneable {
     /** */
     private GridLuceneFile file;
 
@@ -53,6 +54,11 @@ public class GridLuceneInputStream extends IndexInput {
     /** */
     private final GridUnsafeMemory mem;
 
+    /** */
+    private volatile boolean closed;
+
+    /** */
+    private boolean isClone;
     /**
      * Constructor.
      *
@@ -80,7 +86,24 @@ public GridLuceneInputStream(String name, GridLuceneFile f) throws IOException {
 
     /** {@inheritDoc} */
     @Override public void close() {
-        // nothing to do here
+        if (!isClone) {
+            closed = true;
+
+            file.releaseRef();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public IndexInput clone() {
+        GridLuceneInputStream clone = (GridLuceneInputStream) super.clone();
+
+        if(closed)
+            throw new AlreadyClosedException(toString());
+
+        clone.isClone = true;
+
+        return clone;
+
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java
index 8d3d79cd27770..81f2f60671477 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java
@@ -82,6 +82,8 @@ public void reset() {
     /** {@inheritDoc} */
     @Override public void close() throws IOException {
         flush();
+
+        file.releaseRef();
     }
 
     /** {@inheritDoc} */
@@ -174,7 +176,7 @@ private void setFileLength() {
      *
      * @return Bytes used.
      */
-    public long sizeInBytes() {
+    long sizeInBytes() {
         return (long)file.numBuffers() * (long)BUFFER_SIZE;
     }
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java
new file mode 100644
index 0000000000000..a26d5c52d52d9
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import javax.cache.Cache;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.query.Query;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.TextQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cache.query.annotations.QueryTextField;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * FullTest queries left test.
+ */
+public class GridCacheFullTextQuerySelfTest extends GridCommonAbstractTest {
+    /** Cache size. */
+    private static final int MAX_ITEM_COUNT = 100;
+
+    /** Cache name */
+    private static final String PERSON_CACHE = "Person";
+
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(ipFinder);
+
+        cfg.setDiscoverySpi(disco);
+
+        cfg.setIncludeEventTypes();
+
+        cfg.setConnectorConfiguration(null);
+
+        CacheConfiguration<Integer, Person> cacheCfg = defaultCacheConfiguration();
+
+        cacheCfg.setName(PERSON_CACHE)
+            .setCacheMode(PARTITIONED)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+            .setWriteSynchronizationMode(FULL_SYNC)
+            .setBackups(0)
+            .setIndexedTypes(Integer.class, Person.class);
+
+        cfg.setCacheConfiguration(cacheCfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrids(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * JUnit.
+     *
+     * @throws Exception In case of error.
+     */
+    public void testLocalTextQueryWithKeepBinary() throws Exception {
+        checkTextQuery(true, true);
+    }
+
+    /**
+     * JUnit.
+     *
+     * @throws Exception In case of error.
+     */
+    public void testLocalTextQuery() throws Exception {
+        checkTextQuery(true, false);
+    }
+
+    /**
+     * JUnit.
+     *
+     * @throws Exception In case of error.
+     */
+    public void testTextQueryWithKeepBinary() throws Exception {
+        checkTextQuery(false, true);
+    }
+
+    /**
+     * JUnit.
+     *
+     * @throws Exception In case of error.
+     */
+    public void testTextQuery() throws Exception {
+        checkTextQuery(false, false);
+    }
+
+    /**
+     * @param loc local query flag.
+     * @param keepBinary keep binary flag.
+     */
+    private void checkTextQuery(boolean loc, boolean keepBinary) throws Exception {
+        final IgniteEx ignite = grid(0);
+
+        // 1. Populate cache with data, calculating expected count in parallel.
+        Set<Integer> exp = populateCache(ignite, loc, MAX_ITEM_COUNT, new IgnitePredicate<Integer>() {
+            @Override
+            public boolean apply(Integer x) {
+                return String.valueOf(x).startsWith("1");
+            }
+        });
+
+        // 2. Validate results.
+        TextQuery qry = new TextQuery<>(Person.class, "1*").setLocal(loc);
+
+        validateQueryResults(ignite, qry, exp, keepBinary);
+
+        clearCache(ignite);
+    }
+
+    /**
+     * Clear cache with check.
+     */
+    private static void clearCache(IgniteEx ignite) {
+        IgniteCache<Integer, Person> cache = ignite.cache(PERSON_CACHE);
+
+        cache.clear();
+
+        List all = cache.query(new TextQuery<>(Person.class, "1*")).getAll();
+
+        assertTrue(all.isEmpty());
+    }
+
+    /**
+     * Fill cache.
+     *
+     * @throws IgniteCheckedException if failed.
+     */
+    private static Set<Integer> populateCache(IgniteEx ignite, boolean loc, int cnt,
+        IgnitePredicate<Integer> expectedEntryFilter) throws IgniteCheckedException {
+        IgniteInternalCache<Integer, Person> cache = ignite.cachex(PERSON_CACHE);
+
+        assertNotNull(cache);
+
+        Random rand = new Random();
+
+        HashSet<Integer> exp = new HashSet<>();
+
+        Affinity<Integer> aff = cache.affinity();
+
+        ClusterNode localNode = cache.context().localNode();
+
+        for (int i = 0; i < cnt; i++) {
+            int val = rand.nextInt(cnt);
+
+            cache.put(val, new Person(String.valueOf(val), val));
+
+            if (expectedEntryFilter.apply(val) && (!loc || aff.isPrimary(localNode, val)))
+                exp.add(val);
+        }
+
+        return exp;
+    }
+
+    /**
+     * Check query results.
+     *
+     * @throws IgniteCheckedException if failed.
+     */
+    private static void validateQueryResults(IgniteEx ignite, Query qry, Set<Integer> exp,
+        boolean keepBinary) throws IgniteCheckedException {
+        IgniteCache<Integer, Person> cache = ignite.cache(PERSON_CACHE);
+
+        if (keepBinary) {
+            IgniteCache<Integer, BinaryObject> cache0 = cache.withKeepBinary();
+
+            try (QueryCursor<Cache.Entry<Integer, BinaryObject>> cursor = cache0.query(qry)) {
+                Set<Integer> exp0 = new HashSet<>(exp);
+
+                List<Cache.Entry<Integer, ?>> all = new ArrayList<>();
+
+                for (Cache.Entry<Integer, BinaryObject> entry : cursor.getAll()) {
+                    all.add(entry);
+
+                    assertEquals(entry.getKey().toString(), entry.getValue().field("name"));
+
+                    assertEquals(entry.getKey(), entry.getValue().field("age"));
+
+                    exp0.remove(entry.getKey());
+                }
+
+                checkForMissedKeys(ignite, exp0, all);
+            }
+
+            try (QueryCursor<Cache.Entry<Integer, BinaryObject>> cursor = cache0.query(qry)) {
+                Set<Integer> exp0 = new HashSet<>(exp);
+
+                List<Cache.Entry<Integer, ?>> all = new ArrayList<>();
+
+                for (Cache.Entry<Integer, BinaryObject> entry : cursor.getAll()) {
+                    all.add(entry);
+
+                    assertEquals(entry.getKey().toString(), entry.getValue().field("name"));
+
+                    assertEquals(entry.getKey(), entry.getValue().field("age"));
+
+                    exp0.remove(entry.getKey());
+                }
+
+                checkForMissedKeys(ignite, exp0, all);
+            }
+        }
+        else {
+            try (QueryCursor<Cache.Entry<Integer, Person>> cursor = cache.query(qry)) {
+                Set<Integer> exp0 = new HashSet<>(exp);
+
+                List<Cache.Entry<Integer, ?>> all = new ArrayList<>();
+
+                for (Cache.Entry<Integer, Person> entry : cursor.getAll()) {
+                    all.add(entry);
+
+                    assertEquals(entry.getKey().toString(), entry.getValue().name);
+
+                    assertEquals(entry.getKey(), Integer.valueOf(entry.getValue().age));
+
+                    exp0.remove(entry.getKey());
+                }
+
+                checkForMissedKeys(ignite, exp0, all);
+            }
+
+            try (QueryCursor<Cache.Entry<Integer, Person>> cursor = cache.query(qry)) {
+                Set<Integer> exp0 = new HashSet<>(exp);
+
+                List<Cache.Entry<Integer, ?>> all = new ArrayList<>();
+
+                for (Cache.Entry<Integer, Person> entry : cursor.getAll()) {
+                    all.add(entry);
+
+                    assertEquals(entry.getKey().toString(), entry.getValue().name);
+
+                    assertEquals(entry.getKey().intValue(), entry.getValue().age);
+
+                    exp0.remove(entry.getKey());
+                }
+
+                checkForMissedKeys(ignite, exp0, all);
+            }
+        }
+    }
+
+    /**
+     * Check if there is missed keys.
+     *
+     * @throws IgniteCheckedException if failed.
+     */
+    private static void checkForMissedKeys(IgniteEx ignite, Collection<Integer> exp,
+        List<Cache.Entry<Integer, ?>> all) throws IgniteCheckedException {
+        if (exp.size() == 0)
+            return;
+
+        IgniteInternalCache<Integer, Person> cache = ignite.cachex(PERSON_CACHE);
+
+        assertNotNull(cache);
+
+        StringBuilder sb = new StringBuilder();
+
+        Affinity<Integer> aff = cache.affinity();
+
+        for (Integer key : exp) {
+            Integer part = aff.partition(key);
+
+            sb.append(
+                String.format("Query did not return expected key '%d' (exists: %s), partition '%d', partition nodes: ",
+                    key, cache.get(key) != null, part));
+
+            Collection<ClusterNode> partNodes = aff.mapPartitionToPrimaryAndBackups(part);
+
+            for (ClusterNode node : partNodes)
+                sb.append(node).append("  ");
+
+            sb.append(";\n");
+        }
+
+        sb.append("Returned keys: ");
+
+        for (Cache.Entry e : all)
+            sb.append(e.getKey()).append(" ");
+
+        sb.append(";\n");
+
+        fail(sb.toString());
+    }
+
+    /**
+     * Test model class.
+     */
+    public static class Person implements Serializable {
+        /** */
+        @QueryTextField
+        String name;
+
+        /** */
+        @QuerySqlField(index = true)
+        int age;
+
+        /** */
+        @QuerySqlField final Date birthday;
+
+        /**
+         * Constructor
+         */
+        public Person(String name, int age) {
+            this.name = name;
+            this.age = age % 2000;
+
+            Calendar cal = Calendar.getInstance();
+            cal.add(Calendar.YEAR, -age);
+
+            birthday = cal.getTime();
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java
index 5921ba0832f4e..8809631b3d8bf 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java
@@ -117,6 +117,10 @@ public void testFullTextQueryNodeJoin() throws Exception {
 
                 Ignite started = startGrid(GRID_CNT);
 
+                //TODO: remove next line when IGNITE-2229 issue will be fixed.
+                // see https://issues.apache.org/jira/browse/IGNITE-2229
+                awaitPartitionMapExchange();
+
                 for (int i = 0; i < 100; i++) {
                     QueryCursor<Cache.Entry<AffinityKey<Integer>, IndexedEntity>> res = started.cache(null)
                         .query(new TextQuery<AffinityKey<Integer>, IndexedEntity>(IndexedEntity.class, "indexed"));
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 4fb729d56c1ed..6ae26e2c9c096 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.processors.cache.CacheReplicatedQueryMetricsLocalSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheSqlQueryValueCopySelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheCrossCacheQuerySelfTest;
+import org.apache.ignite.internal.processors.cache.GridCacheFullTextQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheQueryIndexDisabledSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheQueryIndexingDisabledSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheQueryInternalKeysSelfTest;
@@ -53,6 +54,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheDistributedJoinTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheDuplicateEntityConfigurationSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheFieldsQueryNoDataSelfTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheFullTextQueryNodeJoiningSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheInsertSqlQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheJoinPartitionedAndReplicatedTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheJoinQueryWithAffinityKeyTest;
@@ -214,6 +216,10 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(CacheQueryOffheapEvictDataLostTest.class);
 
+        // Full text queries.
+        suite.addTestSuite(GridCacheFullTextQuerySelfTest.class);
+        suite.addTestSuite(IgniteCacheFullTextQueryNodeJoiningSelfTest.class);
+
         // Ignite cache and H2 comparison.
         suite.addTestSuite(BaseH2CompareQueryTest.class);
         suite.addTestSuite(H2CompareBigQueryTest.class);

From 8c992fb8ba33a0c0ac5c0fb741ee8ffd515c0f31 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Tue, 1 Aug 2017 17:46:27 +0300
Subject: [PATCH 305/516] IGNITE-5775 Fix removing jobs from activeJobs for
 jobAlwaysActivate

---
 .../ignite/internal/processors/job/GridJobProcessor.java     | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index f88ea07d069bc..7889a5579c7bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -1769,6 +1769,11 @@ private class JobEventListener implements GridJobEventListener {
                 maxFinishedJobsTime.setIfGreater(execTime);
 
                 if (jobAlwaysActivate) {
+                    if (!activeJobs.remove(worker.getJobId(), worker))
+                        cancelledJobs.remove(worker.getJobId(), worker);
+
+                    heldJobs.remove(worker.getJobId());
+                    
                     if (metricsUpdateFreq > -1L)
                         updateJobMetrics();
                 }

From eb3471075035d4d6ed3f54b3c26e929a6a097262 Mon Sep 17 00:00:00 2001
From: Igor Sapego <isapego@gridgain.com>
Date: Tue, 1 Aug 2017 18:30:54 +0300
Subject: [PATCH 306/516] Fix for C++ tests

---
 modules/platforms/cpp/odbc-test/src/queries_test.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index a5793483ed8e9..e13d45c371b26 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -1803,8 +1803,8 @@ BOOST_AUTO_TEST_CASE(TestParamsNum)
 
 BOOST_AUTO_TEST_CASE(TestExecuteAfterCursorClose)
 {
-    TestType in(1, 2, 3, 4, "5", 6.0f, 7.0, true, Guid(8, 9), BinaryUtils::MakeDateGmt(1987, 6, 5),
-        BinaryUtils::MakeTimestampGmt(1998, 12, 27, 1, 2, 3, 456));
+    TestType in(1, 2, 3, 4, "5", 6.0f, 7.0, true, Guid(8, 9), common::MakeDateGmt(1987, 6, 5),
+        common::MakeTimestampGmt(1998, 12, 27, 1, 2, 3, 456));
 
     cache1.Put(1, in);
 

From 9f12a2d4bf333840df50d2a56478349f530b65c8 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 2 Aug 2017 14:23:26 +0300
Subject: [PATCH 307/516] IGNITE-4800: Test fixed.

---
 .../h2/GridIndexingSpiAbstractSelfTest.java   | 157 +++++++++++-------
 1 file changed, 93 insertions(+), 64 deletions(-)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
index d6a5fb1f396e4..3d92e365ee90a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
@@ -22,7 +22,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import org.apache.ignite.IgniteCheckedException;
@@ -35,6 +34,7 @@
 import org.apache.ignite.internal.processors.query.GridQueryIndexType;
 import org.apache.ignite.internal.processors.query.GridQueryProperty;
 import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
+import org.apache.ignite.internal.util.lang.GridCloseableIterator;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -42,6 +42,7 @@
 import org.apache.ignite.plugin.extensions.communication.MessageWriter;
 import org.apache.ignite.spi.IgniteSpiCloseableIterator;
 import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.indexing.IndexingSpi;
 import org.apache.ignite.testframework.GridStringLogger;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -110,7 +111,7 @@ protected void startIndexing(IgniteH2Indexing spi) throws Exception {
      * @param name Name.
      */
     private CacheConfiguration cacheCfg(String name) {
-        CacheConfiguration<?,?> cfg = new CacheConfiguration<>();
+        CacheConfiguration<?, ?> cfg = new CacheConfiguration<>();
 
         cfg.setName(name);
 
@@ -230,16 +231,16 @@ public void testSpi() throws Exception {
         assertEquals(0, spi.size(typeAB.space(), typeAB));
         assertEquals(0, spi.size(typeBA.space(), typeBA));
 
-        assertFalse(spi.queryLocalSql(typeAA.space(), "select * from A.A", null, Collections.emptySet(), typeAA.name(), null, null).hasNext());
-        assertFalse(spi.queryLocalSql(typeAB.space(), "select * from A.B", null, Collections.emptySet(), typeAB.name(), null, null).hasNext());
-        assertFalse(spi.queryLocalSql(typeBA.space(), "select * from B.A", null, Collections.emptySet(), typeBA.name(), null, null).hasNext());
+        assertFalse(hasLocalQueryResults(spi, typeAA.space(), "select * from A.A", typeAA.name()));
+        assertFalse(hasLocalQueryResults(spi, typeAB.space(), "select * from A.B", typeAB.name()));
+        assertFalse(hasLocalQueryResults(spi, typeBA.space(), "select * from B.A", typeBA.name()));
 
-        assertFalse(spi.queryLocalSql(typeBA.space(), "select * from B.A, A.B, A.A", null,
-            Collections.emptySet(), typeBA.name(), null, null).hasNext());
+        assertFalse(hasLocalQueryResults(spi, typeBA.space(), "select * from B.A, A.B, A.A", typeBA.name()));
 
-        try {
-            spi.queryLocalSql(typeBA.space(), "select aa.*, ab.*, ba.* from A.A aa, A.B ab, B.A ba", null,
-                Collections.emptySet(), typeBA.name(), null, null).hasNext();
+        try (GridCloseableIterator<IgniteBiTuple<Object, Object>> res =
+                 spi.queryLocalSql(typeBA.space(), "select aa.*, ab.*, ba.* from A.A aa, A.B ab, B.A ba", null,
+                     Collections.emptySet(), typeBA.name(), null, null)) {
+            res.hasNext();
 
             fail("Enumerations of aliases in select block must be prohibited");
         }
@@ -247,11 +248,9 @@ public void testSpi() throws Exception {
             // all fine
         }
 
-        assertFalse(spi.queryLocalSql(typeAB.space(), "select ab.* from A.B ab", null,
-            Collections.emptySet(), typeAB.name(), null, null).hasNext());
+        assertFalse(hasLocalQueryResults(spi, typeAB.space(), "select ab.* from A.B ab", typeAB.name()));
 
-        assertFalse(spi.queryLocalSql(typeBA.space(), "select   ba.*   from B.A  as ba", null,
-            Collections.emptySet(), typeBA.name(), null, null).hasNext());
+        assertFalse(hasLocalQueryResults(spi, typeBA.space(), "select   ba.*   from B.A  as ba", typeBA.name()));
 
         // Nothing to remove.
         spi.remove("A", key(1), aa(1, "", 10));
@@ -304,80 +303,92 @@ public void testSpi() throws Exception {
         assertEquals(1, spi.size(typeBA.space(), typeBA));
 
         // Query data.
-        Iterator<IgniteBiTuple<Integer, Map<String, Object>>> res =
-            spi.queryLocalSql(typeAA.space(), "from a order by age", null, Collections.emptySet(), typeAA.name(), null, null);
-
-        assertTrue(res.hasNext());
-        assertEquals(aa(3, "Borya", 18).value(null, false), value(res.next()));
-        assertTrue(res.hasNext());
-        assertEquals(aa(2, "Valera", 19).value(null, false), value(res.next()));
-        assertFalse(res.hasNext());
+        try (GridCloseableIterator<IgniteBiTuple<Integer, Map<String, Object>>> res =
+                 spi.queryLocalSql(typeAA.space(), "from a order by age", null, Collections.emptySet(),
+                     typeAA.name(), null, null)) {
 
-        res = spi.queryLocalSql(typeAA.space(), "select aa.* from a aa order by aa.age", null,
-            Collections.emptySet(), typeAA.name(), null, null);
+            assertTrue(res.hasNext());
+            assertEquals(aa(3, "Borya", 18).value(null, false), value(res.next()));
+            assertTrue(res.hasNext());
+            assertEquals(aa(2, "Valera", 19).value(null, false), value(res.next()));
+            assertFalse(res.hasNext());
+        }
 
-        assertTrue(res.hasNext());
-        assertEquals(aa(3, "Borya", 18).value(null, false), value(res.next()));
-        assertTrue(res.hasNext());
-        assertEquals(aa(2, "Valera", 19).value(null, false), value(res.next()));
-        assertFalse(res.hasNext());
+        try (GridCloseableIterator<IgniteBiTuple<Integer, Map<String, Object>>> res =
+                 spi.queryLocalSql(typeAA.space(), "select aa.* from a aa order by aa.age", null,
+                     Collections.emptySet(), typeAA.name(), null, null)) {
 
-        res = spi.queryLocalSql(typeAB.space(), "from b order by name", null, Collections.emptySet(), typeAB.name(), null, null);
+            assertTrue(res.hasNext());
+            assertEquals(aa(3, "Borya", 18).value(null, false), value(res.next()));
+            assertTrue(res.hasNext());
+            assertEquals(aa(2, "Valera", 19).value(null, false), value(res.next()));
+            assertFalse(res.hasNext());
+        }
 
-        assertTrue(res.hasNext());
-        assertEquals(ab(1, "Vasya", 20, "Some text about Vasya goes here.").value(null, false), value(res.next()));
-        assertTrue(res.hasNext());
-        assertEquals(ab(4, "Vitalya", 20, "Very Good guy").value(null, false), value(res.next()));
-        assertFalse(res.hasNext());
+        try (GridCloseableIterator<IgniteBiTuple<Integer, Map<String, Object>>> res =
+                 spi.queryLocalSql(typeAB.space(), "from b order by name", null, Collections.emptySet(),
+                     typeAB.name(), null, null)) {
 
-        res = spi.queryLocalSql(typeAB.space(), "select bb.* from b as bb order by bb.name", null,
-            Collections.emptySet(), typeAB.name(), null, null);
+            assertTrue(res.hasNext());
+            assertEquals(ab(1, "Vasya", 20, "Some text about Vasya goes here.").value(null, false), value(res.next()));
+            assertTrue(res.hasNext());
+            assertEquals(ab(4, "Vitalya", 20, "Very Good guy").value(null, false), value(res.next()));
+            assertFalse(res.hasNext());
+        }
 
-        assertTrue(res.hasNext());
-        assertEquals(ab(1, "Vasya", 20, "Some text about Vasya goes here.").value(null, false), value(res.next()));
-        assertTrue(res.hasNext());
-        assertEquals(ab(4, "Vitalya", 20, "Very Good guy").value(null, false), value(res.next()));
-        assertFalse(res.hasNext());
+        try (GridCloseableIterator<IgniteBiTuple<Integer, Map<String, Object>>> res =
+                 spi.queryLocalSql(typeAB.space(), "select bb.* from b as bb order by bb.name", null,
+                     Collections.emptySet(), typeAB.name(), null, null)) {
 
+            assertTrue(res.hasNext());
+            assertEquals(ab(1, "Vasya", 20, "Some text about Vasya goes here.").value(null, false), value(res.next()));
+            assertTrue(res.hasNext());
+            assertEquals(ab(4, "Vitalya", 20, "Very Good guy").value(null, false), value(res.next()));
+            assertFalse(res.hasNext());
+        }
 
-        res = spi.queryLocalSql(typeBA.space(), "from a", null, Collections.emptySet(), typeBA.name(), null, null);
+        try (GridCloseableIterator<IgniteBiTuple<Integer, Map<String, Object>>> res =
+                 spi.queryLocalSql(typeBA.space(), "from a", null, Collections.emptySet(), typeBA.name(), null, null)) {
 
-        assertTrue(res.hasNext());
-        assertEquals(ba(2, "Kolya", 25, true).value(null, false), value(res.next()));
-        assertFalse(res.hasNext());
+            assertTrue(res.hasNext());
+            assertEquals(ba(2, "Kolya", 25, true).value(null, false), value(res.next()));
+            assertFalse(res.hasNext());
+        }
 
         // Text queries
-        Iterator<IgniteBiTuple<Integer, Map<String, Object>>> txtRes = spi.queryLocalText(typeAB.space(), "good",
-            typeAB, null);
+        try (GridCloseableIterator<IgniteBiTuple<Integer, Map<String, Object>>> txtRes = spi.queryLocalText(typeAB.space(), "good",
+            typeAB, null)) {
 
-        assertTrue(txtRes.hasNext());
-        assertEquals(ab(4, "Vitalya", 20, "Very Good guy").value(null, false), value(txtRes.next()));
-        assertFalse(txtRes.hasNext());
+            assertTrue(txtRes.hasNext());
+            assertEquals(ab(4, "Vitalya", 20, "Very Good guy").value(null, false), value(txtRes.next()));
+            assertFalse(txtRes.hasNext());
+        }
 
         // Fields query
         GridQueryFieldsResult fieldsRes =
             spi.queryLocalSqlFields("A", "select a.a.name n1, a.a.age a1, b.a.name n2, " +
-            "b.a.age a2 from a.a, b.a where a.a.id = b.a.id ", Collections.emptySet(), null, false, 0, null);
+                "b.a.age a2 from a.a, b.a where a.a.id = b.a.id ", Collections.emptySet(), null, false, 0, null);
 
         String[] aliases = {"N1", "A1", "N2", "A2"};
-        Object[] vals = { "Valera", 19, "Kolya", 25};
+        Object[] vals = {"Valera", 19, "Kolya", 25};
 
-        IgniteSpiCloseableIterator<List<?>> it = fieldsRes.iterator();
+        try (IgniteSpiCloseableIterator<List<?>> it = fieldsRes.iterator()) {
 
-        assertTrue(it.hasNext());
+            assertTrue(it.hasNext());
 
-        List<?> fields = it.next();
+            List<?> fields = it.next();
 
-        assertEquals(4, fields.size());
+            assertEquals(4, fields.size());
 
-        int i = 0;
+            int i = 0;
 
-        for (Object f : fields) {
-            assertEquals(aliases[i], fieldsRes.metaData().get(i).fieldName());
-            assertEquals(vals[i++], f);
-        }
+            for (Object f : fields) {
+                assertEquals(aliases[i], fieldsRes.metaData().get(i).fieldName());
+                assertEquals(vals[i++], f);
+            }
 
-        assertFalse(it.hasNext());
+            assertFalse(it.hasNext());
+        }
 
         // Remove
         spi.remove(typeAA.space(), key(2), aa(2, "Valera", 19));
@@ -430,6 +441,23 @@ public void testSpi() throws Exception {
         assertEquals(-1, spi.size(typeAA.space(), typeAA));
     }
 
+    /**
+     * @param spi {@link IndexingSpi} implementation
+     * @param spaceName Space name.
+     * @param qry SQL query string.
+     * @param type Query return type.
+     * @return {@code False} if resultset is empty, {@code True} otherwise.
+     * @throws IgniteCheckedException
+     */
+    private static boolean hasLocalQueryResults(IgniteH2Indexing spi, String spaceName, String qry,
+        String type) throws IgniteCheckedException {
+
+        try (GridCloseableIterator<IgniteBiTuple<Object, Object>> res =
+                 spi.queryLocalSql(spaceName, qry, null, Collections.emptySet(), type, null, null)) {
+            return res.hasNext();
+        }
+    }
+
     /**
      * Test long queries write explain warnings into log.
      *
@@ -610,7 +638,8 @@ String space() {
 
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
-        @Override public void setValue(String field, Object key, Object val, Object propVal) throws IgniteCheckedException {
+        @Override public void setValue(String field, Object key, Object val,
+            Object propVal) throws IgniteCheckedException {
             assert !F.isEmpty(field);
 
             assert key instanceof Integer;

From f0f1c82c8f9877d952f639ffe56803043e53415b Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Fri, 4 Aug 2017 14:58:52 +0300
Subject: [PATCH 308/516] IGNITE-5290 Partial back port commit from master
 (42293fa sboikov <sboikov@gridgain.com> on 29.05.2017 at 16:41)

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../CacheContinuousQueryHandler.java          | 152 +++++---
 .../CacheContinuousQueryListener.java         |   5 +
 .../CacheContinuousQueryManager.java          |  50 ++-
 .../continuous/GridContinuousProcessor.java   |   2 -
 .../IgniteCacheMessageWriteTimeoutTest.java   |   2 +
 ...ousQueryConcurrentPartitionUpdateTest.java | 341 ++++++++++++++++++
 ...ntinuousQueryFailoverAbstractSelfTest.java | 116 +-----
 ...CacheContinuousQueryOrderingEventTest.java |  21 ++
 .../IgniteCacheQuerySelfTestSuite3.java       |   3 +
 9 files changed, 513 insertions(+), 179 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 17f4308c0709b..165b8b77e0ab4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -141,7 +141,7 @@ public class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler
     private transient boolean keepBinary;
 
     /** */
-    private transient ConcurrentMap<Integer, PartitionRecovery> rcvs;
+    private transient ConcurrentMap<Integer, PartitionRecovery> rcvs = new ConcurrentHashMap<>();
 
     /** */
     private transient ConcurrentMap<Integer, EntryBuffer> entryBufs;
@@ -340,8 +340,6 @@ public void keepBinary(boolean keepBinary) {
 
         ackBuf = new AcknowledgeBuffer();
 
-        rcvs = new ConcurrentHashMap<>();
-
         this.nodeId = nodeId;
 
         this.routineId = routineId;
@@ -490,6 +488,16 @@ public void keepBinary(boolean keepBinary) {
                 sendBackupAcknowledge(ackBuf.acknowledgeOnTimeout(), routineId, ctx);
             }
 
+            @Override public void checkQueueOnTimeout(GridKernalContext ctx) {
+                for (PartitionRecovery rcv : rcvs.values()) {
+                    Collection<CacheEntryEvent<? extends K, ? extends V>> evts = rcv.onTimeout(
+                        ctx.cache().safeJcache(cacheName, cacheId), ctx.cache().context().cacheContext(cacheId));
+
+                    if (!evts.isEmpty())
+                        locLsnr.onUpdated(evts);
+                }
+            }
+
             @Override public void skipUpdateEvent(CacheContinuousQueryEvent<K, V> evt,
                 AffinityTopologyVersion topVer, boolean primary) {
                 assert evt != null;
@@ -557,20 +565,6 @@ private void prepareEntry(GridCacheContext cctx, UUID nodeId, CacheContinuousQue
             entry.prepareMarshal(cctx);
     }
 
-    /**
-     * Wait topology.
-     */
-    public void waitTopologyFuture(GridKernalContext ctx) throws IgniteCheckedException {
-        GridCacheContext<K, V> cctx = cacheContext(ctx);
-
-        if (!cctx.isLocal()) {
-            cacheContext(ctx).affinity().affinityReadyFuture(initTopVer).get();
-
-            for (int partId = 0; partId < cacheContext(ctx).affinity().partitions(); partId++)
-                getOrCreatePartitionRecovery(ctx, partId);
-        }
-    }
-
     /** {@inheritDoc} */
     @Override public void unregister(UUID routineId, GridKernalContext ctx) {
         assert routineId != null;
@@ -864,31 +858,7 @@ private String taskName() {
         PartitionRecovery rec = rcvs.get(partId);
 
         if (rec == null) {
-            Long partCntr = null;
-
-            AffinityTopologyVersion initTopVer0 = initTopVer;
-
-            if (initTopVer0 != null) {
-                GridCacheContext<K, V> cctx = cacheContext(ctx);
-
-                GridCacheAffinityManager aff = cctx.affinity();
-
-                if (initUpdCntrsPerNode != null) {
-                    for (ClusterNode node : aff.nodes(partId, initTopVer)) {
-                        Map<Integer, Long> map = initUpdCntrsPerNode.get(node.id());
-
-                        if (map != null) {
-                            partCntr = map.get(partId);
-
-                            break;
-                        }
-                    }
-                }
-                else if (initUpdCntrs != null)
-                    partCntr = initUpdCntrs.get(partId);
-            }
-
-            rec = new PartitionRecovery(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY), initTopVer0, partCntr);
+            rec = new PartitionRecovery(ctx.log(CU.CONTINUOUS_QRY_LOG_CATEGORY));
 
             PartitionRecovery oldRec = rcvs.putIfAbsent(partId, rec);
 
@@ -936,7 +906,7 @@ private CacheContinuousQueryEntry handleEntry(CacheContinuousQueryEntry e) {
     /**
      *
      */
-    private static class PartitionRecovery {
+    private static class PartitionRecovery<K, V> {
         /** Event which means hole in sequence. */
         private static final CacheContinuousQueryEntry HOLE = new CacheContinuousQueryEntry();
 
@@ -949,6 +919,12 @@ private static class PartitionRecovery {
         /** */
         private long lastFiredEvt;
 
+        /** */
+        private long prevCheckCntr = -1;
+
+        /** */
+        private int pendingSize;
+
         /** */
         private AffinityTopologyVersion curTop = AffinityTopologyVersion.NONE;
 
@@ -957,19 +933,9 @@ private static class PartitionRecovery {
 
         /**
          * @param log Logger.
-         * @param topVer Topology version.
-         * @param initCntr Update counters.
          */
-        PartitionRecovery(IgniteLogger log, AffinityTopologyVersion topVer, @Nullable Long initCntr) {
+        PartitionRecovery(IgniteLogger log) {
             this.log = log;
-
-            if (initCntr != null) {
-                assert topVer.topologyVersion() > 0 : topVer;
-
-                this.lastFiredEvt = initCntr;
-
-                curTop = topVer;
-            }
         }
 
         /**
@@ -1067,6 +1033,17 @@ private static class PartitionRecovery {
                         }
                     }
                 }
+                else if (!entry.isBackup() &&
+                    (entry.topologyVersion() != null && entry.topologyVersion().equals(curTop))) {
+
+                    if (log.isDebugEnabled())
+                        log.debug("Processed message out of order: " + entry);
+
+                    return !entry.isFiltered() ?
+                        F.<CacheEntryEvent<? extends K, ? extends V>>
+                            asList(new CacheContinuousQueryEvent<K, V>(cache, cctx, entry)) :
+                        Collections.<CacheEntryEvent<? extends K, ? extends V>>emptyList();
+                }
                 else {
                     if (log.isDebugEnabled())
                         log.debug("Skip duplicate continuous query message: " + entry);
@@ -1139,6 +1116,73 @@ private static class PartitionRecovery {
 
             return entries;
         }
+
+        /**
+         * Check queue on stuck.
+         *
+         * @param cache Cache.
+         * @param cctx Cache context.
+         *
+         * @return Stacked events.
+         */
+        Collection<CacheEntryEvent<? extends K, ? extends V>> onTimeout(IgniteCache cache, GridCacheContext cctx) {
+            List<CacheEntryEvent<? extends K, ? extends V>> res = Collections.emptyList();
+
+            synchronized (pendingEvts) {
+                // Visited this the first time.
+                if (prevCheckCntr == -1) {
+                    prevCheckCntr = lastFiredEvt;
+                    pendingSize = pendingEvts.size();
+
+                    return Collections.emptyList();
+                }
+
+                // Pending event not found.
+                if (pendingEvts.size() == 0) {
+                    prevCheckCntr = lastFiredEvt;
+                    pendingSize = 0;
+
+                    if (log.isDebugEnabled())
+                        log.debug("No stuck events.");
+
+                    return Collections.emptyList();
+                }
+
+                if (log.isDebugEnabled())
+                    log.debug("Check stuck events [lastFiredEvt=" + lastFiredEvt + ", "
+                        + "prevCheckCntr=" + prevCheckCntr + ", "
+                        + "pendingSize=" + pendingSize + ", "
+                        + "pendingEvts=" + pendingEvts + "]");
+
+                if (pendingSize > 0 && prevCheckCntr == lastFiredEvt) {
+                    int pendingSize0 = Math.min(pendingSize, pendingEvts.size());
+
+                    assert pendingSize0 > 0;
+
+                    res = new ArrayList<>(pendingSize0);
+                    Iterator<Map.Entry<Long, CacheContinuousQueryEntry>> iter = pendingEvts.entrySet().iterator();
+
+                    for (int i = 0; i < pendingSize0; i++) {
+                        Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
+
+                        if (e.getValue() != HOLE && !e.getValue().isFiltered())
+                            res.add(new CacheContinuousQueryEvent(cache, cctx, e.getValue()));
+
+                        iter.remove();
+
+                        lastFiredEvt = e.getKey();
+                    }
+                }
+
+                prevCheckCntr = lastFiredEvt;
+                pendingSize = pendingEvts.size();
+
+                if (log.isDebugEnabled())
+                    log.debug("Process stuck events [lastFiredEvt=" + lastFiredEvt + ", " + "evts=" + res + "]");
+            }
+
+            return res;
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
index 84b22f933e848..c9b486a5d9617 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
@@ -68,6 +68,11 @@ public void onEntryUpdated(CacheContinuousQueryEvent<K, V> evt, boolean primary,
      */
     public void acknowledgeBackupOnTimeout(GridKernalContext ctx);
 
+    /**
+     * @param ctx Context.
+     */
+    public void checkQueueOnTimeout(GridKernalContext ctx);
+
     /**
      * @param evt Event
      * @param topVer Topology version.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 91c199184a2de..a5f647aa3d2b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -99,6 +99,9 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
     /** */
     private static final long BACKUP_ACK_FREQ = 5000;
 
+    /** */
+    private static final long QUEUE_CHECKER_FREQ = 3000;
+
     /** Listeners. */
     private final ConcurrentMap<UUID, CacheContinuousQueryListener> lsnrs = new ConcurrentHashMap8<>();
 
@@ -139,6 +142,8 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
 
             cctx.time().schedule(new BackupCleaner(lsnrs, cctx.kernalContext()), BACKUP_ACK_FREQ, BACKUP_ACK_FREQ);
         }
+
+        cctx.time().schedule(new QueueChecker(lsnrs, cctx.kernalContext()), QUEUE_CHECKER_FREQ, QUEUE_CHECKER_FREQ);
     }
 
     /** {@inheritDoc} */
@@ -655,18 +660,6 @@ private UUID executeQuery0(CacheEntryUpdatedListener locLsnr,
             autoUnsubscribe,
             pred).get();
 
-        try {
-            if (hnd.isQuery() && cctx.userCache() && !onStart)
-                hnd.waitTopologyFuture(cctx.kernalContext());
-        }
-        catch (IgniteCheckedException e) {
-            log.warning("Failed to start continuous query.", e);
-
-            cctx.kernalContext().continuous().stopRoutine(id);
-
-            throw new IgniteCheckedException("Failed to start continuous query.", e);
-        }
-
         if (notifyExisting) {
             final Iterator<GridCacheEntryEx> it = cctx.cache().allEntries().iterator();
 
@@ -1196,8 +1189,37 @@ public BackupCleaner(Map<UUID, CacheContinuousQueryListener> lsnrs, GridKernalCo
 
         /** {@inheritDoc} */
         @Override public void run() {
-            for (CacheContinuousQueryListener lsnr : lsnrs.values())
-                lsnr.acknowledgeBackupOnTimeout(ctx);
+            if (!lsnrs.isEmpty()) {
+                for (CacheContinuousQueryListener lsnr : lsnrs.values())
+                    lsnr.acknowledgeBackupOnTimeout(ctx);
+            }
+        }
+    }
+
+    /**
+     * Task flash backup queue.
+     */
+    private static final class QueueChecker implements Runnable {
+        /** Listeners. */
+        private final Map<UUID, CacheContinuousQueryListener> lsnrs;
+
+        /** Context. */
+        private final GridKernalContext ctx;
+
+        /**
+         * @param lsnrs Listeners.
+         */
+        QueueChecker(Map<UUID, CacheContinuousQueryListener> lsnrs, GridKernalContext ctx) {
+            this.lsnrs = lsnrs;
+            this.ctx = ctx;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            if (!lsnrs.isEmpty()) {
+                for (CacheContinuousQueryListener lsnr : lsnrs.values())
+                    lsnr.checkQueueOnTimeout(ctx);
+            }
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index f0429bc3a3c91..50d29d1f8de6e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -240,8 +240,6 @@ public GridContinuousProcessor(GridKernalContext ctx) {
 
                                 if (cctx != null && cntrsPerNode != null && !cctx.isLocal() && cctx.affinityNode())
                                     cntrsPerNode.put(ctx.localNodeId(), cctx.topology().updateCounters(false));
-
-                                routine.handler().updateCounters(topVer, cntrsPerNode, cntrs);
                             }
 
                             fut.onRemoteRegistered();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index 6256225f0a54e..339e39ac4e98b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -70,6 +70,8 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testMessageQueueLimit() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-12398");
+
         startGridsMultiThreaded(3);
 
         for (int i = 0; i < 15; i++) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
new file mode 100644
index 0000000000000..c1675f328d706
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryUpdatedListener;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.query.CacheQueryEntryEvent;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static String DEFAULT_CACHE_NAME = "cache-name";
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi) cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatePartitionAtomic() throws Exception {
+        concurrentUpdatePartition(ATOMIC, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatePartitionTx() throws Exception {
+        concurrentUpdatePartition(TRANSACTIONAL, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatePartitionAtomicClient() throws Exception {
+        concurrentUpdatePartition(ATOMIC, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatePartitionTxClient() throws Exception {
+        concurrentUpdatePartition(TRANSACTIONAL, true);
+    }
+
+    /**
+     * @param atomicityMode Cache atomicity mode.
+     * @param clientMode Client mode.
+     * @throws Exception If failed.
+     */
+    private void concurrentUpdatePartition(CacheAtomicityMode atomicityMode, boolean clientMode) throws Exception {
+        Ignite srv = startGrid(0);
+
+        client = clientMode;
+
+        Ignite client = startGrid(1);
+
+        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+        ccfg.setAtomicityMode(atomicityMode);
+
+        IgniteCache clientCache = client.createCache(ccfg);
+
+        final AtomicInteger evtCnt = new AtomicInteger();
+
+        Affinity<Integer> aff = srv.affinity(DEFAULT_CACHE_NAME);
+
+        final List<Integer> keys = new ArrayList<>();
+
+        final int KEYS = 1000;
+
+        for (int i = 0; i < 10_000_000; i++) {
+            if (aff.partition(i) == 0) {
+                keys.add(i);
+
+                if (keys.size() == KEYS)
+                    break;
+            }
+        }
+
+        assertEquals(KEYS, keys.size());
+
+        final int THREADS = 10;
+        final int UPDATES = 1000;
+
+        final IgniteCache<Object, Object> cache0 = clientMode ? client.cache(DEFAULT_CACHE_NAME)
+            : srv.cache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < 10; i++) {
+            log.info("Iteration: " + i);
+
+            ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+            final ConcurrentSkipListSet<Long> sets = new ConcurrentSkipListSet<>();
+
+            qry.setLocalListener(new CacheEntryUpdatedListener<Object, Object>() {
+                @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts) {
+                    for (CacheEntryEvent evt : evts) {
+                        assertNotNull(evt.getKey());
+                        assertNotNull(evt.getValue());
+
+                        CacheQueryEntryEvent e = (CacheQueryEntryEvent)evt.unwrap(CacheQueryEntryEvent.class);
+
+                        sets.add(e.getPartitionUpdateCounter());
+
+                        evtCnt.incrementAndGet();
+                    }
+                }
+            });
+
+            QueryCursor qryCur = clientCache.query(qry);
+
+            GridTestUtils.runMultiThreaded(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                    for (int i = 0; i < UPDATES; i++)
+                        cache0.put(keys.get(rnd.nextInt(KEYS)), i);
+
+                    return null;
+                }
+            }, THREADS, "update");
+
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    log.info("Events: " + evtCnt.get());
+
+                    return evtCnt.get() >= THREADS * UPDATES;
+                }
+            }, 5000);
+
+            assertEquals(THREADS * UPDATES, evtCnt.get());
+
+            qryCur.close();
+
+            evtCnt.set(0);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatesAndQueryStartAtomic() throws Exception {
+        concurrentUpdatesAndQueryStart(ATOMIC);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentUpdatesAndQueryStartTx() throws Exception {
+        concurrentUpdatesAndQueryStart(TRANSACTIONAL);
+    }
+
+    /**
+     * @param atomicityMode Cache atomicity mode.
+     * @throws Exception If failed.
+     */
+    private void concurrentUpdatesAndQueryStart(CacheAtomicityMode atomicityMode) throws Exception {
+        Ignite srv = startGrid(0);
+
+        client = true;
+
+        Ignite client = startGrid(1);
+
+        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+        ccfg.setAtomicityMode(atomicityMode);
+
+        IgniteCache clientCache = this.client ? client.createCache(ccfg) : srv.createCache(ccfg);
+
+        Affinity<Integer> aff = srv.affinity(DEFAULT_CACHE_NAME);
+
+        final List<Integer> keys = new ArrayList<>();
+
+        final int KEYS = 1_000;
+
+        for (int i = 0; i < 10_000_000; i++) {
+            if (aff.partition(i) == 0) {
+                keys.add(i);
+
+                if (keys.size() == KEYS)
+                    break;
+            }
+        }
+
+        assertEquals(KEYS, keys.size());
+
+        final int THREADS = 10;
+        final int UPDATES = 100;
+
+        for (int i = 0; i < 5; i++) {
+            log.info("Iteration: " + i);
+
+            ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+            final AtomicInteger evtCnt = new AtomicInteger();
+
+            final ConcurrentSkipListSet<Long> sets = new ConcurrentSkipListSet<>();
+
+            qry.setLocalListener(new CacheEntryUpdatedListener<Object, Object>() {
+                @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts) {
+                    for (CacheEntryEvent evt : evts) {
+                        assertNotNull(evt.getKey());
+                        assertNotNull(evt.getValue());
+
+                        CacheQueryEntryEvent e = (CacheQueryEntryEvent)evt.unwrap(CacheQueryEntryEvent.class);
+
+                        sets.add(e.getPartitionUpdateCounter());
+
+                        if ((Integer)evt.getValue() >= 0)
+                            evtCnt.incrementAndGet();
+                    }
+                }
+            });
+
+            QueryCursor cur;
+
+            final IgniteCache<Object, Object> srvCache = srv.cache(DEFAULT_CACHE_NAME);
+
+            final AtomicBoolean stop = new AtomicBoolean();
+
+            try {
+                IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+                    @Override public Void call() throws Exception {
+                        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                        while (!stop.get())
+                            srvCache.put(keys.get(rnd.nextInt(KEYS)), rnd.nextInt(100) - 200);
+
+                        return null;
+                    }
+                }, THREADS, "update");
+
+                U.sleep(1000);
+
+                cur = clientCache.query(qry);
+
+                U.sleep(1000);
+
+                stop.set(true);
+
+                fut.get();
+            }
+            finally {
+                stop.set(true);
+            }
+
+            GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                    for (int i = 0; i < UPDATES; i++)
+                        srvCache.put(keys.get(rnd.nextInt(KEYS)), i);
+
+                    return null;
+                }
+            }, THREADS, "update");
+
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    log.info("Events: " + evtCnt.get());
+
+                    return evtCnt.get() >= THREADS * UPDATES;
+                }
+            }, 10000);
+
+            if ((THREADS * UPDATES) != evtCnt.get())
+                log.info("Entries: " + sets);
+
+            assertEquals(THREADS * UPDATES, evtCnt.get());
+
+            cur.close();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
index d2cb710edbeb5..ffb7557ff1258 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
@@ -142,7 +142,6 @@ public abstract class CacheContinuousQueryFailoverAbstractSelfTest extends GridC
         TestCommunicationSpi commSpi = new TestCommunicationSpi();
 
         commSpi.setSharedMemoryPort(-1);
-        commSpi.setIdleConnectionTimeout(100);
 
         cfg.setCommunicationSpi(commSpi);
 
@@ -1387,7 +1386,7 @@ public void testBackupQueueCleanupClientQuery() throws Exception {
             @Override public boolean apply() {
                 return backupQueue.isEmpty();
             }
-        }, 2000);
+        }, 10000);
 
         assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.size() < BACKUP_ACK_THRESHOLD);
 
@@ -1409,13 +1408,13 @@ public void testBackupQueueCleanupClientQuery() throws Exception {
             @Override public boolean apply() {
                 return backupQueue.isEmpty();
             }
-        }, ACK_FREQ + 2000);
-
-        assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.isEmpty());
+        }, ACK_FREQ + 20000);
 
         if (!latch.await(5, SECONDS))
             fail("Failed to wait for notifications [exp=" + keys.size() + ", left=" + lsnr.latch.getCount() + ']');
 
+        assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.isEmpty());
+
         cur.close();
 
         assertFalse("Unexpected error during test, see log for details.", err);
@@ -1465,9 +1464,7 @@ public void testBackupQueueEvict() throws Exception {
             @Override public boolean apply() {
                 return backupQueue.isEmpty();
             }
-        }, 2000);
-
-        assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.size() < BACKUP_ACK_THRESHOLD);
+        }, 20000);
 
         boolean wait = waitForCondition(new GridAbsPredicate() {
             @Override public boolean apply() {
@@ -1477,6 +1474,8 @@ public void testBackupQueueEvict() throws Exception {
 
         assertTrue("Entry evicted.", wait);
 
+        assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.size() < BACKUP_ACK_THRESHOLD);
+
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
             @Override public boolean apply() {
                 return backupQueue.isEmpty();
@@ -2076,107 +2075,6 @@ public void testMultiThreadedFailover() throws Exception {
         assertFalse("Unexpected error during test, see log for details.", err);
     }
 
-    /**
-     * @throws Exception If failed.
-     */
-    public void testMultiThreaded() throws Exception {
-        this.backups = 2;
-
-        final int SRV_NODES = 3;
-
-        startGridsMultiThreaded(SRV_NODES);
-
-        client = true;
-
-        Ignite qryClient = startGrid(SRV_NODES);
-
-        final IgniteCache<Object, Object> cache = qryClient.cache(null);
-
-        CacheEventListener1 lsnr = new CacheEventListener1(true);
-
-        ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
-
-        qry.setLocalListener(lsnr);
-
-        QueryCursor<?> cur = cache.query(qry);
-
-        client = false;
-
-        final int SRV_IDX = SRV_NODES - 1;
-
-        List<Integer> keys = primaryKeys(ignite(SRV_IDX).cache(null), 10);
-
-        final int THREADS = 10;
-
-        for (int i = 0; i < keys.size(); i++) {
-            log.info("Iteration: " + i);
-
-            Ignite srv = ignite(SRV_IDX);
-
-            TestCommunicationSpi spi = (TestCommunicationSpi)srv.configuration().getCommunicationSpi();
-
-            spi.sndFirstOnly = new AtomicBoolean(false);
-
-            final Integer key = keys.get(i);
-
-            final AtomicInteger val = new AtomicInteger();
-
-            CountDownLatch latch = new CountDownLatch(THREADS);
-
-            lsnr.latch = latch;
-
-            IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
-                @Override public Object call() throws Exception {
-                    Integer val0 = val.getAndIncrement();
-
-                    cache.put(key, val0);
-
-                    return null;
-                }
-            }, THREADS, "update-thread");
-
-            fut.get();
-
-            stopGrid(SRV_IDX);
-
-            if (!latch.await(5, SECONDS))
-                fail("Failed to wait for notifications [exp=" + THREADS + ", left=" + lsnr.latch.getCount() + ']');
-
-            assertEquals(THREADS, lsnr.allEvts.size());
-
-            Set<Integer> vals = new HashSet<>();
-
-            boolean err = false;
-
-            for (CacheEntryEvent<?, ?> evt : lsnr.allEvts) {
-                assertEquals(key, evt.getKey());
-                assertNotNull(evt.getValue());
-
-                if (!vals.add((Integer)evt.getValue())) {
-                    err = true;
-
-                    log.info("Extra event: " + evt);
-                }
-            }
-
-            for (int v = 0; v < THREADS; v++) {
-                if (!vals.contains(v)) {
-                    err = true;
-
-                    log.info("Event for value not received: " + v);
-                }
-            }
-
-            assertFalse("Invalid events, see log for details.", err);
-
-            lsnr.allEvts.clear();
-
-            startGrid(SRV_IDX);
-        }
-
-        cur.close();
-    }
-
     /**
      * @param logAll If {@code true} logs all unexpected values.
      * @param expEvts Expected values.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
index 7d975f2a28209..4dcdd32745ac0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOrderingEventTest.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.Cache;
 import javax.cache.configuration.FactoryBuilder;
 import javax.cache.event.CacheEntryEvent;
 import javax.cache.event.CacheEntryUpdatedListener;
@@ -49,6 +50,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteAsyncCallback;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
@@ -391,6 +393,25 @@ protected void doOrderingTest(
                 }
             }
 
+            IgniteCache<Object, Object> cache = grid(0).cache(ccfg.getName());
+
+            for (int i = 0; i < KEYS; i++) {
+                cache.put(new QueryTestKey(i), new QueryTestValue(-1));
+
+                cache.remove(new QueryTestValue(i));
+            }
+
+            GridTestUtils.waitForCondition(new PA() {
+                @Override public boolean apply() {
+                    return qryCntr.get() >= 4 * KEYS * LISTENER_CNT * NODES;
+                }
+            }, 3000L);
+
+            for (BlockingQueue<CacheEntryEvent<QueryTestKey, QueryTestValue>> q : rcvdEvts)
+                q.clear();
+
+            qryCntr.set(0);
+
             IgniteInternalFuture<Long> f = GridTestUtils.runMultiThreadedAsync(new Runnable() {
                 @Override public void run() {
                     ThreadLocalRandom rnd = ThreadLocalRandom.current();
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
index 032dd3b5d8426..47de0ff842418 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchAckTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchForceServerModeAckTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFilterListenerTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryConcurrentPartitionUpdateTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryExecuteInPrimaryTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryAsyncFilterRandomOperationTest;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryFilterRandomOperationTest;
@@ -122,6 +123,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheKeepBinaryIterationNearEnabledTest.class);
         suite.addTestSuite(IgniteCacheContinuousQueryBackupQueueTest.class);
 
+        suite.addTestSuite(CacheContinuousQueryConcurrentPartitionUpdateTest.class);
+
         return suite;
     }
 }

From 3d64cf1a023ce71cf522b6ce843669fcf44a3145 Mon Sep 17 00:00:00 2001
From: Ivan Rakov <ivan.glukos@gmail.com>
Date: Thu, 30 Mar 2017 14:56:40 +0300
Subject: [PATCH 309/516] IGNITE-3477 - Fixed concurrent read-through

(cherry picked from commit 31ffef4)
---
 .../ignite/internal/processors/cache/GridCacheAdapter.java  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 7c5a54dfcb852..7aa48199bb28a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2014,9 +2014,13 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
                                         CacheObject cacheVal = ctx.toCacheObject(val);
 
                                         while (true) {
-                                            GridCacheEntryEx entry = entryEx(key);
+                                            GridCacheEntryEx entry = null;
 
                                             try {
+                                                entry = entryEx(key);
+
+                                                entry.unswap();
+
                                                 EntryGetResult verVal = entry.versionedValue(
                                                     cacheVal,
                                                     res.version(),

From 9b9eabd97494f291444f42159cb5977b407dda8c Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 9 Aug 2017 18:58:02 +0700
Subject: [PATCH 310/516] IGNITE-5987 Added -nq (visor will not quit in batch
 mode) option for Visor Cmd.

---
 .../apache/ignite/visor/commands/VisorConsole.scala  | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
index ad8c2edaaf8dd..31b4ff06b00dc 100644
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
@@ -93,6 +93,7 @@ class VisorConsole {
             println("        -cfg=<path>          - connect with specified configuration.")
             println("        -b=<path>            - batch mode with file.")
             println("        -e=cmd1;cmd2;...     - batch mode with commands.")
+            println("        -nq                  - batch mode will not quit after execution (useful for alerts monitoring).")
 
             visor.quit()
         }
@@ -104,6 +105,10 @@ class VisorConsole {
         val cfgFile = argValue("cfg", argLst)
         val batchFile = argValue("b", argLst)
         val batchCommand = argValue("e", argLst)
+        val noBatchQuit = hasArgName("nq", argLst)
+
+        if (noBatchQuit && batchFile.isEmpty && batchCommand.isEmpty)
+            visor.warn("Option \"-nq\" will be ignored because batch mode options \"-b\" or \"-e\" were not specified.")
 
         cfgFile.foreach(cfg => {
             if (cfg.trim.isEmpty) {
@@ -150,7 +155,10 @@ class VisorConsole {
             case Some(cmd) =>
                 visor.batchMode = true
 
-                new ByteArrayInputStream((cmd + "\nquit\n").getBytes("UTF-8"))
+                val script = if (noBatchQuit) cmd else cmd + "\nquit\n"
+
+                new ByteArrayInputStream(script.getBytes("UTF-8"))
+
             case None => new FileInputStream(FileDescriptor.in)
         }
 
@@ -160,7 +168,7 @@ class VisorConsole {
 
             new TerminalSupport(false) {}
         } catch {
-            case ignored: ClassNotFoundException => null
+            case _: ClassNotFoundException => null
         }
 
         val reader = new ConsoleReader(inputStream, System.out, term)

From 737260b070c3178d91ef04f7b200da87c9a1874f Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 10 Aug 2017 18:54:57 +0300
Subject: [PATCH 311/516] IGNITE-4991 Do not print out system properties when
 IGNITE_TO_STRING_INCLUDE_SENSITIVE is set to false

---
 .../main/java/org/apache/ignite/internal/IgniteKernal.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 9bada6c35ce3b..a1997b1adfed7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2213,7 +2213,7 @@ public <K, V> GridCacheAdapter<K, V> internalCache(@Nullable String name) {
     private void ackSystemProperties() {
         assert log != null;
 
-        if (log.isDebugEnabled())
+        if (log.isDebugEnabled() && S.INCLUDE_SENSITIVE)
             for (Map.Entry<Object, Object> entry : snapshot().entrySet())
                 log.debug("System property [" + entry.getKey() + '=' + entry.getValue() + ']');
     }
@@ -2384,7 +2384,7 @@ private void ackVmArguments(RuntimeMXBean rtBean) {
         assert log != null;
 
         // Ack IGNITE_HOME and VM arguments.
-        if (log.isInfoEnabled()) {
+        if (log.isInfoEnabled() && S.INCLUDE_SENSITIVE) {
             log.info("IGNITE_HOME=" + cfg.getIgniteHome());
             log.info("VM arguments: " + rtBean.getInputArguments());
         }

From bad6c4cfe4c315b2418161e667a395176404d295 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Thu, 10 Aug 2017 18:54:57 +0300
Subject: [PATCH 312/516] IGNITE-4991 Do not print out system properties when
 IGNITE_TO_STRING_INCLUDE_SENSITIVE is set to false

---
 .../main/java/org/apache/ignite/internal/IgniteKernal.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 7506a9125351a..ed588c4d5e7b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2257,7 +2257,7 @@ public <K, V> GridCacheAdapter<K, V> internalCache(@Nullable String name) {
     private void ackSystemProperties() {
         assert log != null;
 
-        if (log.isDebugEnabled())
+        if (log.isDebugEnabled() && S.INCLUDE_SENSITIVE)
             for (Map.Entry<Object, Object> entry : snapshot().entrySet())
                 log.debug("System property [" + entry.getKey() + '=' + entry.getValue() + ']');
     }
@@ -2428,7 +2428,7 @@ private void ackVmArguments(RuntimeMXBean rtBean) {
         assert log != null;
 
         // Ack IGNITE_HOME and VM arguments.
-        if (log.isInfoEnabled()) {
+        if (log.isInfoEnabled() && S.INCLUDE_SENSITIVE) {
             log.info("IGNITE_HOME=" + cfg.getIgniteHome());
             log.info("VM arguments: " + rtBean.getInputArguments());
         }

From d4f4a323a80193dffabff96b45d0f44e5ee22d9b Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Mon, 14 Aug 2017 16:44:51 +0300
Subject: [PATCH 313/516] Merge branch 'ignite-1.7.14' into ignite-1.8.10

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../CacheContinuousQueryConcurrentPartitionUpdateTest.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
index c1675f328d706..36d1dc44aabf4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryConcurrentPartitionUpdateTest.java
@@ -82,7 +82,7 @@ public class CacheContinuousQueryConcurrentPartitionUpdateTest extends GridCommo
     /**
      * @throws Exception If failed.
      */
-    public void testConcurrentUpdatePartitionAtomic() throws Exception {
+    public void _testConcurrentUpdatePartitionAtomic() throws Exception {
         concurrentUpdatePartition(ATOMIC, false);
     }
 
@@ -96,7 +96,7 @@ public void testConcurrentUpdatePartitionTx() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testConcurrentUpdatePartitionAtomicClient() throws Exception {
+    public void _testConcurrentUpdatePartitionAtomicClient() throws Exception {
         concurrentUpdatePartition(ATOMIC, true);
     }
 
@@ -205,7 +205,7 @@ private void concurrentUpdatePartition(CacheAtomicityMode atomicityMode, boolean
     /**
      * @throws Exception If failed.
      */
-    public void testConcurrentUpdatesAndQueryStartAtomic() throws Exception {
+    public void _testConcurrentUpdatesAndQueryStartAtomic() throws Exception {
         concurrentUpdatesAndQueryStart(ATOMIC);
     }
 

From b7bf1c09e8404d10eac57c13c9c6720c040d0c8e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 14 Jul 2017 20:14:47 +0300
Subject: [PATCH 314/516] IGNITE-5452: GridTimeoutProcessor can hang on stop.
 This closes #2279.

(cherry picked from commit b95c261)
---
 .../timeout/GridTimeoutProcessor.java         |  18 +-
 .../IgniteTxRemoveTimeoutObjectsTest.java     | 194 ++++++++++++++++++
 .../timeout/GridTimeoutProcessorSelfTest.java |  68 ++++--
 .../testsuites/IgniteCacheTestSuite3.java     |   3 +
 4 files changed, 265 insertions(+), 18 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index 0bbf9c376fc60..7fabd5cadb6a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -37,7 +37,7 @@
  */
 public class GridTimeoutProcessor extends GridProcessorAdapter {
     /** */
-    private final IgniteThread timeoutWorker;
+    private final TimeoutWorker timeoutWorker;
 
     /** Time-based sorted set for timeout objects. */
     private final GridConcurrentSkipListSet<GridTimeoutObject> timeoutObjs =
@@ -62,13 +62,12 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
     public GridTimeoutProcessor(GridKernalContext ctx) {
         super(ctx);
 
-        timeoutWorker = new IgniteThread(ctx.config().getGridName(), "grid-timeout-worker",
-            new TimeoutWorker());
+        timeoutWorker = new TimeoutWorker();
     }
 
     /** {@inheritDoc} */
     @Override public void start() {
-        timeoutWorker.start();
+        new IgniteThread(timeoutWorker).start();
 
         if (log.isDebugEnabled())
             log.debug("Timeout processor started.");
@@ -76,7 +75,7 @@ public GridTimeoutProcessor(GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public void stop(boolean cancel) throws IgniteCheckedException {
-        U.interrupt(timeoutWorker);
+        timeoutWorker.cancel();
         U.join(timeoutWorker);
 
         if (log.isDebugEnabled())
@@ -159,6 +158,13 @@ private class TimeoutWorker extends GridWorker {
                             timeoutObj.onTimeout();
                         }
                         catch (Throwable e) {
+                            if (isCancelled() && !(e instanceof Error)){
+                                if (log.isDebugEnabled())
+                                    log.debug("Error when executing timeout callback: " + timeoutObj);
+
+                                return;
+                            }
+
                             U.error(log, "Error when executing timeout callback: " + timeoutObj, e);
 
                             if (e instanceof Error)
@@ -170,7 +176,7 @@ private class TimeoutWorker extends GridWorker {
                 }
 
                 synchronized (mux) {
-                    while (true) {
+                    while (!isCancelled()) {
                         // Access of the first element must be inside of
                         // synchronization block, so we don't miss out
                         // on thread notification events sent from
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java
new file mode 100644
index 0000000000000..bdb73ad303448
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionTimeoutException;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
+
+/**
+ * Test correctness of rollback a transaction with timeout during the grid stop.
+ */
+public class IgniteTxRemoveTimeoutObjectsTest extends GridCacheAbstractSelfTest {
+    /** */
+    private static final int PUT_CNT = 1000;
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 3;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 60_000;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxRemoveTimeoutObjects() throws Exception {
+        IgniteCache<Integer, Integer> cache0 = grid(0).cache(null);
+        IgniteCache<Integer, Integer> cache1 = grid(1).cache(null);
+
+        // start additional grid to be closed.
+        IgniteCache<Integer, Integer> cacheAdditional = startGrid(gridCount()).cache(null);
+
+        for (int i = 0; i < PUT_CNT; i++)
+            cache0.put(i, Integer.MAX_VALUE);
+
+        logTimeoutObjectsFrequency();
+
+        info("Tx1 started");
+        try (Transaction tx = grid(gridCount()).transactions().txStart(PESSIMISTIC, SERIALIZABLE, 100, PUT_CNT)) {
+            try {
+                for (int i = 0; i < PUT_CNT; i++) {
+                    cacheAdditional.put(i, Integer.MIN_VALUE);
+
+                    if (i % 100 == 0)
+                        logTimeoutObjectsFrequency();
+                }
+
+                U.sleep(200);
+
+                tx.commit();
+
+                fail("A timeout should have happened.");
+            }
+            catch (Exception e) {
+                assertTrue(X.hasCause(e, TransactionTimeoutException.class));
+            }
+        }
+
+        assertDoesNotContainLockTimeoutObjects();
+
+        logTimeoutObjectsFrequency();
+
+        stopGrid(gridCount());
+
+        awaitPartitionMapExchange();
+
+        info("Grid2 closed.");
+
+        assertDoesNotContainLockTimeoutObjects();
+
+        logTimeoutObjectsFrequency();
+
+        // Check that the values have not changed and lock can be acquired.
+        try (Transaction tx2 = grid(1).transactions().txStart(PESSIMISTIC, SERIALIZABLE)) {
+            info("Tx2 started");
+
+            for (int i = 0; i < PUT_CNT; i++) {
+                assertEquals(cache1.get(i).intValue(), Integer.MAX_VALUE);
+                cache1.put(i, i);
+
+                if (i % (PUT_CNT / 5) == 0)
+                    logTimeoutObjectsFrequency();
+            }
+
+            tx2.commit();
+        }
+
+        info("Tx2 stopped");
+
+        // Check that that changes committed.
+        for (int i = 0; i < PUT_CNT; i++)
+            assertEquals(cache0.get(i).intValue(), i);
+    }
+
+    /**
+     * Fails if at least one grid contains LockTimeoutObjects.
+     */
+    private void assertDoesNotContainLockTimeoutObjects() {
+        for (Ignite ignite : G.allGrids()) {
+            for (GridTimeoutObject object : getTimeoutObjects((IgniteEx)ignite)) {
+                if (object.getClass().getSimpleName().equals("LockTimeoutObject"))
+                    fail("Grids contain LockTimeoutObjects.");
+            }
+        }
+    }
+
+    /**
+     * Print the number of each timeout object type on each grid to the log.
+     */
+    private void logTimeoutObjectsFrequency() {
+        StringBuilder sb = new StringBuilder("Timeout objects frequency [");
+
+        for (Ignite ignite : G.allGrids()) {
+            IgniteEx igniteEx = (IgniteEx)ignite;
+
+            Map<String, Integer> objFreqMap = new HashMap<>();
+
+            Set<GridTimeoutObject> objs = getTimeoutObjects(igniteEx);
+
+            for (GridTimeoutObject obj : objs) {
+                String clsName = obj.getClass().getSimpleName();
+
+                Integer cnt = objFreqMap.get(clsName);
+
+                if (cnt == null)
+                    objFreqMap.put(clsName, 1);
+                else
+                    objFreqMap.put(clsName, cnt + 1);
+            }
+
+            sb.append("[")
+                .append(igniteEx.name()).append(": size=")
+                .append(objs.size()).append(", ");
+
+            for (Map.Entry<String, Integer> entry : objFreqMap.entrySet()) {
+                sb.append(entry.getKey()).append("=")
+                    .append(entry.getValue())
+                    .append(", ");
+            }
+
+            sb.delete(sb.length() - 2, sb.length())
+                .append("]; ");
+        }
+
+        sb.delete(sb.length() - 2, sb.length())
+            .append("]");
+
+        info(sb.toString()
+            .replaceAll("distributed.IgniteTxRollbackOnStopTest", "Grid"));
+    }
+
+    /**
+     * @param igniteEx IgniteEx.
+     * @return Set of timeout objects that process on current IgniteEx.
+     */
+    private Set<GridTimeoutObject> getTimeoutObjects(IgniteEx igniteEx) {
+        GridTimeoutProcessor timeout = igniteEx.context().timeout();
+
+        return GridTestUtils.getFieldValue(timeout, timeout.getClass(), "timeoutObjs");
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessorSelfTest.java
index eb248cfd5dced..606b10252b19a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessorSelfTest.java
@@ -40,6 +40,11 @@ public class GridTimeoutProcessorSelfTest extends GridCommonAbstractTest {
     /** Kernal context. */
     private GridTestKernalContext ctx;
 
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 60_000;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         ctx = newContext();
@@ -84,7 +89,9 @@ public void testTimeouts() throws Exception {
                 }
 
                 /** {@inheritDoc} */
-                @Override public long endTime() { return endTime; }
+                @Override public long endTime() {
+                    return endTime;
+                }
 
                 /** {@inheritDoc} */
                 @Override public void onTimeout() {
@@ -152,10 +159,14 @@ public void testTimeoutsMultithreaded() throws Exception {
                         private final long endTime = System.currentTimeMillis() + RAND.nextInt(1000) + 500;
 
                         /** {@inheritDoc} */
-                        @Override public IgniteUuid timeoutId() { return id; }
+                        @Override public IgniteUuid timeoutId() {
+                            return id;
+                        }
 
                         /** {@inheritDoc} */
-                        @Override public long endTime() { return endTime; }
+                        @Override public long endTime() {
+                            return endTime;
+                        }
 
                         /** {@inheritDoc} */
                         @Override public void onTimeout() {
@@ -307,9 +318,8 @@ public void testTimeoutNeverCalled() throws Exception {
         assert timeObjs.size() == max;
 
         // Remove timeout objects so that they aren't able to times out (supposing the cycle takes less than 500 ms).
-        for (GridTimeoutObject obj : timeObjs) {
+        for (GridTimeoutObject obj : timeObjs)
             ctx.timeout().removeTimeoutObject(obj);
-        }
 
         Thread.sleep(1000);
 
@@ -350,7 +360,9 @@ public void testTimeoutNeverCalledMultithreaded() throws Exception {
                         }
 
                         /** {@inheritDoc} */
-                        @Override public long endTime() { return endTime; }
+                        @Override public long endTime() {
+                            return endTime;
+                        }
 
                         /** {@inheritDoc} */
                         @Override public void onTimeout() {
@@ -370,9 +382,8 @@ public void testTimeoutNeverCalledMultithreaded() throws Exception {
 
                 // Remove timeout objects so that they aren't able to times out
                 // (supposing the cycle takes less than 500 ms).
-                for (GridTimeoutObject obj : timeObjs) {
+                for (GridTimeoutObject obj : timeObjs)
                     ctx.timeout().removeTimeoutObject(obj);
-                }
             }
         }, threads, "timeout-test-worker");
 
@@ -381,6 +392,9 @@ public void testTimeoutNeverCalledMultithreaded() throws Exception {
         assert callCnt.get() == 0;
     }
 
+    /**
+     * @throws Exception If test failed.
+     */
     public void testAddRemoveInterleaving() throws Exception {
         final AtomicInteger callCnt = new AtomicInteger(0);
 
@@ -430,9 +444,8 @@ public void testAddRemoveInterleaving() throws Exception {
 
                 // Remove timeout objects so that they aren't able to times out
                 // (supposing the cycle takes less than 500 ms).
-                for (GridTimeoutObject obj : timeObjs) {
+                for (GridTimeoutObject obj : timeObjs)
                     ctx.timeout().removeTimeoutObject(obj);
-                }
             }
         }, 100, "timeout-test-worker");
 
@@ -516,10 +529,14 @@ public void testTimeoutCallOnce() throws Exception {
             private int cnt;
 
             /** {@inheritDoc} */
-            @Override public IgniteUuid timeoutId() { return id; }
+            @Override public IgniteUuid timeoutId() {
+                return id;
+            }
 
             /** {@inheritDoc} */
-            @Override public long endTime() { return endTime; }
+            @Override public long endTime() {
+                return endTime;
+            }
 
             /** {@inheritDoc} */
             @Override public void onTimeout() {
@@ -608,4 +625,31 @@ public void testTimeoutSameEndTime() throws Exception {
 
         assert latch.await(3000, MILLISECONDS);
     }
+
+    /**
+     * Test that eaten {@link InterruptedException} will not hang on the closing of the grid.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testCancelingWithClearedInterruptedFlag() throws Exception {
+        final CountDownLatch onTimeoutCalled = new CountDownLatch(1);
+
+        ctx.timeout().addTimeoutObject(new GridTimeoutObjectAdapter(10) {
+            /** {@inheritDoc} */
+            @Override public void onTimeout() {
+                try {
+                    onTimeoutCalled.countDown();
+
+                    // Wait for CacheProcessor has stopped and cause InterruptedException
+                    // which clears interrupted flag.
+                    Thread.sleep(Long.MAX_VALUE);
+                }
+                catch (InterruptedException ignore) {
+                    // No-op.
+                }
+            }
+        });
+
+        onTimeoutCalled.await();
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index 0785714c61b0d..87051c3d425ce 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.CacheAsyncOperationsTest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheMixedModeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteTxGetAfterStopTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteTxRemoveTimeoutObjectsTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDaemonNodePartitionedSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedOnlyP2PDisabledByteArrayValuesSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedOnlyP2PEnabledByteArrayValuesSelfTest;
@@ -205,6 +206,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(CacheAsyncOperationsTest.class);
 
+        suite.addTestSuite(IgniteTxRemoveTimeoutObjectsTest.class);
+
         return suite;
     }
 }

From b63c7911adedd0952edb5c2062a2dad8927976ac Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Tue, 15 Aug 2017 17:36:02 +0300
Subject: [PATCH 315/516] Muted
 IgniteCacheMessageWriteTimeoutTest#testMessageQueueLimit test.

---
 .../cache/distributed/IgniteCacheMessageWriteTimeoutTest.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index 0dd4079733081..8f3f7723efb60 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -75,7 +75,9 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testMessageQueueLimit() throws Exception {
-        for (int i = 0; i < 3; i++) {
+        fail("https://ggsystems.atlassian.net/browse/GG-12398");
+
+        for (int i = 0; i < 15; i++) {
             log.info("Iteration: " + i);
 
             startGridsMultiThreaded(3);

From 60e2de794b52cc7fe3525c5e6de831106bcb9d5f Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Wed, 16 Aug 2017 00:24:07 +0300
Subject: [PATCH 316/516] IgniteCacheNearRestartRollbackSelfTest#testRestarts
 is muted.

---
 .../distributed/IgniteCacheNearRestartRollbackSelfTest.java     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index 3f242b5726867..aea4d7782d858 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -132,6 +132,8 @@ protected CacheConfiguration<Object, Object> cacheConfiguration(String gridName)
      */
     @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
     public void testRestarts() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-12398");
+
         startGrids(4);
 
         Ignite tester = ignite(3);

From 928d445f63cf47ad52185b73ccfef559f22717e8 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Wed, 16 Aug 2017 14:06:37 +0300
Subject: [PATCH 317/516] Fixed flaky test "IgniteCacheEntryListener*"

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../query/continuous/CacheContinuousQueryEntry.java      | 9 +--------
 .../query/continuous/CacheContinuousQueryHandler.java    | 2 --
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index 366a1e05fa46c..ffbbc2c1e05ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -199,13 +199,6 @@ void markFiltered() {
         depInfo = null;
     }
 
-    /**
-     * @param topVer Topology version.
-     */
-    void topologyVersion(AffinityTopologyVersion topVer) {
-        this.topVer = topVer;
-    }
-
     /**
      * @return Size include this event and filtered.
      */
@@ -222,7 +215,7 @@ CacheContinuousQueryEntry forBackupQueue() {
             return this;
 
         CacheContinuousQueryEntry e =
-            new CacheContinuousQueryEntry(cacheId, null, null, null, null, keepBinary, part, updateCntr, null);
+            new CacheContinuousQueryEntry(cacheId, null, null, null, null, keepBinary, part, updateCntr, topVer);
 
         e.flags = flags;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 7cf7d0761db13..26c361498fbab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -481,8 +481,6 @@ public void keepBinary(boolean keepBinary) {
                             for (CacheContinuousQueryEntry e : backupQueue0) {
                                 if (!e.isFiltered())
                                     prepareEntry(cctx, nodeId, e);
-
-                                e.topologyVersion(topVer);
                             }
 
                             ctx.continuous().addBackupNotification(nodeId, routineId, backupQueue0, topic);

From aa5ca8fd154a853c37c2121d88a92e3404eead3a Mon Sep 17 00:00:00 2001
From: Nikolay Izhikov <nizhikov.dev@gmail.com>
Date: Wed, 16 Aug 2017 15:45:16 +0300
Subject: [PATCH 318/516] IGNITE-5897 Fix session init/end logic. This fixes
 tests.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../cache/store/CacheStoreManager.java        |  4 +++-
 .../store/GridCacheStoreManagerAdapter.java   | 20 +++++++++----------
 .../cache/transactions/IgniteTxAdapter.java   |  9 +++++++--
 3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
index 8d6b63dce3c8a..b096edf6a8b95 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
@@ -165,9 +165,11 @@ public boolean removeAll(@Nullable IgniteInternalTx tx, Collection<Object> keys)
     /**
      * @param tx Transaction.
      * @param commit Commit.
+     * @param last {@code True} if this is last store in transaction.
+     * @param storeSessionEnded {@code True} if session for underlying store already ended.
      * @throws IgniteCheckedException If failed.
      */
-    public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException;
+    public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last, boolean storeSessionEnded) throws IgniteCheckedException;
 
     /**
      * End session initiated by write-behind store.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 685e8f6e03181..14ec92237c3d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -777,7 +777,8 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public final void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException {
+    @Override public final void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last,
+        boolean storeSessionEnded) throws IgniteCheckedException {
         assert store != null;
 
         sessionInit0(tx);
@@ -788,7 +789,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
                     lsnr.onSessionEnd(locSes, commit);
             }
 
-            if (!sesHolder.get().ended(store))
+            if (!sesHolder.get().ended(store) && !storeSessionEnded)
                 store.sessionEnd(commit);
         }
         catch (Throwable e) {
@@ -857,7 +858,7 @@ private void sessionInit0(@Nullable IgniteInternalTx tx) throws IgniteCheckedExc
         sesHolder.set(ses);
 
         try {
-            if (sesLsnrs != null && !ses.started(this)) {
+            if (!ses.started(store) && sesLsnrs != null) {
                 for (CacheStoreSessionListener lsnr : sesLsnrs)
                     lsnr.onSessionStart(locSes);
             }
@@ -920,11 +921,8 @@ private static class SessionData {
         private Object attachment;
 
         /** */
-        private final Set<CacheStoreManager> started =
-            new GridSetWrapper<>(new IdentityHashMap<CacheStoreManager, Object>());
-
-        /** */
-        private final Set<CacheStore> ended = new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
+        private final Set<CacheStore> started =
+            new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
 
         /**
          * @param tx Current transaction.
@@ -987,8 +985,8 @@ private void cacheName(String cacheName) {
         /**
          * @return If session is started.
          */
-        private boolean started(CacheStoreManager mgr) {
-            return !started.add(mgr);
+        private boolean started(CacheStore store) {
+            return !started.add(store);
         }
 
         /**
@@ -996,7 +994,7 @@ private boolean started(CacheStoreManager mgr) {
          * @return Whether session already ended on this store instance.
          */
         private boolean ended(CacheStore store) {
-            return !ended.add(store);
+            return !started.remove(store);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 18c301124405f..cd5babe7118b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -40,6 +41,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -61,6 +63,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
+import org.apache.ignite.internal.util.GridSetWrapper;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
 import org.apache.ignite.internal.util.lang.GridTuple;
@@ -1238,13 +1241,15 @@ protected boolean isWriteToStoreFromDhtValid(Collection<CacheStoreManager> store
      * @param commit Commit flag.
      * @throws IgniteCheckedException In case of error.
      */
-    protected void sessionEnd(Collection<CacheStoreManager> stores, boolean commit) throws IgniteCheckedException {
+    protected void sessionEnd(final Collection<CacheStoreManager> stores, boolean commit) throws IgniteCheckedException {
         Iterator<CacheStoreManager> it = stores.iterator();
 
+        Set<CacheStore> visited = new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
+
         while (it.hasNext()) {
             CacheStoreManager store = it.next();
 
-            store.sessionEnd(this, commit, !it.hasNext());
+            store.sessionEnd(this, commit, !it.hasNext(), !visited.add(store.store()));
         }
     }
 

From fdd74de0c3635e12f74d70564bea9945776bfbca Mon Sep 17 00:00:00 2001
From: Alexander Fedotov <alexander.fedotoff@gmail.com>
Date: Wed, 16 Aug 2017 18:03:57 +0300
Subject: [PATCH 319/516] IGNITE-GG-12549 SQL: Add support for Java 8 Time API
 classes in date\time functions

- add tests
---
 ...yWithJsr310Java8DateTimeApiFieldsTest.java | 409 ++++++++++++++++++
 ...heQueryJsr310Java8DateTimeApiBaseTest.java |  71 +++
 ...Jsr310Java8DateTimeApiKeyAndValueTest.java | 301 +++++++++++++
 ...sr310Java8DateTimeApiSupportTestSuite.java |  40 ++
 parent/pom.xml                                |  42 ++
 5 files changed, 863 insertions(+)
 create mode 100644 modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryEntityWithJsr310Java8DateTimeApiFieldsTest.java
 create mode 100644 modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiBaseTest.java
 create mode 100644 modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiKeyAndValueTest.java
 create mode 100644 modules/indexing/src/test/java8/org/apache/ignite/testsuites/CacheQueryJsr310Java8DateTimeApiSupportTestSuite.java

diff --git a/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryEntityWithJsr310Java8DateTimeApiFieldsTest.java b/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryEntityWithJsr310Java8DateTimeApiFieldsTest.java
new file mode 100644
index 0000000000000..045fa3e40788a
--- /dev/null
+++ b/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryEntityWithJsr310Java8DateTimeApiFieldsTest.java
@@ -0,0 +1,409 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2;
+
+import java.io.Serializable;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.List;
+import java.util.Objects;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cache.query.annotations.QuerySqlFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Tests queries against entities with JSR-310 Java 8 Date and Time API fields.
+ */
+public class CacheQueryEntityWithJsr310Java8DateTimeApiFieldsTest extends CacheQueryJsr310Java8DateTimeApiBaseTest {
+    /**
+     * Entity containing JSR-310 fields.
+     */
+    private static class EntityWithJsr310Fields implements Serializable {
+
+        /** Serial version UID. */
+        private static final long serialVersionUID = 1L;
+
+        /** ID. */
+        @QuerySqlField(index = true)
+        private Long id;
+
+        /** {@link LocalTime} field. */
+        @QuerySqlField(index = true)
+        private LocalTime locTime;
+
+        /** {@link LocalDate} field. */
+        @QuerySqlField(index = true)
+        private LocalDate locDate;
+
+        /** {@link LocalDateTime} field. */
+        @QuerySqlField(index = true)
+        private LocalDateTime locDateTime;
+
+        /**
+         * Default constructor.
+         */
+        EntityWithJsr310Fields() {
+        }
+
+        /**
+         * Copy constructor.
+         *
+         * @param entity Entity to copy from.
+         */
+        EntityWithJsr310Fields(EntityWithJsr310Fields entity) {
+            id = entity.id;
+            locTime = LocalTime.from(entity.locTime);
+            locDate = LocalDate.from(entity.locDate);
+            locDateTime = LocalDateTime.from(entity.locDateTime);
+        }
+
+        /**
+         * Constructor.
+         *
+         * @param id ID.
+         * @param locTime {@link LocalTime} value.
+         * @param locDate {@link LocalDate} value.
+         * @param locDateTime {@link LocalDateTime} value.
+         */
+        EntityWithJsr310Fields(Long id, LocalTime locTime, LocalDate locDate, LocalDateTime locDateTime) {
+            this.id = id;
+            this.locTime = locTime;
+            this.locDate = locDate;
+            this.locDateTime = locDateTime;
+        }
+
+        /**
+         * Returns the ID.
+         *
+         * @return ID.
+         */
+        public Long getId() {
+            return id;
+        }
+
+        /**
+         * Sets the ID.
+         *
+         * @param id ID.
+         */
+        public void setId(Long id) {
+            this.id = id;
+        }
+
+        /**
+         * Returns the {@link LocalDateTime} field value
+         *
+         * @return {@link LocalDateTime} field value;
+         */
+        public LocalDateTime getLocalDateTime() {
+            return locDateTime;
+        }
+
+        /**
+         * Returns the {@link LocalDateTime} field value.
+         *
+         * @param locDateTime {@link LocalDateTime} value.
+         */
+        public void setLocalDateTime(LocalDateTime locDateTime) {
+            this.locDateTime = locDateTime;
+        }
+
+        /**
+         * Returns the {@link LocalDate} field value.
+         *
+         * @return {@link LocalDate} field value.
+         */
+        public LocalDate getLocalDate() {
+            return locDate;
+        }
+
+        /**
+         * Sets the {@link LocalDate} field value.
+         *
+         * @param locDate {@link LocalDate} value.
+         */
+        public void setLocalDate(LocalDate locDate) {
+            this.locDate = locDate;
+        }
+
+        /**
+         * Returns the {@link LocalTime} field value.
+         *
+         * @return {@link LocalTime} field value.
+         */
+        public LocalTime getLocalTime() {
+            return locTime;
+        }
+
+        /**
+         * Sets the {@link LocalTime} field value.
+         *
+         * @param locTime {@link LocalTime} value.
+         */
+        public void setLocalTime(LocalTime locTime) {
+            this.locTime = locTime;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            EntityWithJsr310Fields fields = (EntityWithJsr310Fields)o;
+
+            return Objects.equals(id, fields.id) && Objects.equals(locDateTime, fields.locDateTime) &&
+                Objects.equals(locDate, fields.locDate) && Objects.equals(locTime, fields.locTime);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return Objects.hash(id, locDateTime, locDate, locTime);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "EntityWithJsr310Fields{" + "id=" + id + ", locDateTime=" + locDateTime + ", locDate=" + locDate +
+                ", locTime=" + locTime + '}';
+        }
+    }
+
+    /**
+     * Custom SQL functions.
+     */
+    public static class SqlFuncitons {
+        /**
+         * Converts the specified {@link LocalDateTime} object to a {@link Timestamp} object.
+         *
+         * @param localDateTime {@link LocalDateTime} object to convert.
+         * @return Converted object.
+         */
+        @QuerySqlFunction(deterministic = true)
+        public static Timestamp localDateTimeToTimestamp(LocalDateTime localDateTime) {
+            return Timestamp.valueOf(localDateTime);
+        }
+    }
+
+    /** Cache. */
+    private IgniteCache<Long, EntityWithJsr310Fields> cache;
+
+    /** Entity with JSR-310 fields instance. */
+    private final EntityWithJsr310Fields entity =
+        new EntityWithJsr310Fields(1L, LOCAL_TIME, LOCAL_DATE, LOCAL_DATE_TIME);
+
+    /**
+     * Creates a cache configuration.
+     *
+     * @return Cache configuration.
+     */
+    private static CacheConfiguration<Long, EntityWithJsr310Fields> createCacheConfig() {
+        return CacheQueryEntityWithJsr310Java8DateTimeApiFieldsTest.<Long, EntityWithJsr310Fields>createCacheConfig(
+            "entityWithJava8DataTimeFields", Long.class, EntityWithJsr310Fields.class
+        ).setSqlFunctionClasses(SqlFuncitons.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        Ignite ignite = startGridsMultiThreaded(1, true);
+        cache = ignite.getOrCreateCache(createCacheConfig());
+
+        cache.put(entity.getId(), entity);
+    }
+
+    /**
+     * Tests insertion of an entity.
+     *
+     * @throws Exception If failed.
+     */
+    public void testInsertEntityFields() throws Exception {
+        cache.remove(entity.getId());
+
+        assertEquals(0, cache.size());
+
+        SqlFieldsQuery qry = new SqlFieldsQuery(
+            "insert into EntityWithJsr310Fields(_key, id, locTime, locDate, locDateTime) values(?, ?, ?, ?, ?)"
+        ).setArgs(
+            entity.getId(), entity.getId(), entity.getLocalTime(), entity.getLocalDate(), entity.getLocalDateTime()
+        );
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(1L, qryResults.get(0).get(0));
+        assertEquals(1, cache.size());
+        assertEquals(entity, cache.get(entity.getId()));
+    }
+
+    /**
+     * Tests selection of an entity.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSelectEntity() throws Exception {
+        SqlFieldsQuery qry = new SqlFieldsQuery("select _val from EntityWithJsr310Fields");
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(entity, qryResults.get(0).get(0));
+    }
+
+    /**
+     * Tests that DATEDIFF SQL function fails for {@link LocalDateTime}
+     * fields with the time part set to midnight.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDateDiffForLocalDateTimeFieldAtMidnightThrows() throws Exception {
+        GridTestUtils.assertThrows(
+            log,
+            () -> {
+                SqlFieldsQuery qry = new SqlFieldsQuery(
+                    "select DATEDIFF('DAY', locDateTime, CURRENT_DATE ()) from EntityWithJsr310Fields");
+
+                List<List<?>> qryResults = cache.query(qry).getAll();
+
+                assertEquals(1, qryResults.size());
+                assertTrue((Long)qryResults.get(0).get(0) >= DAYS_BEFORE_NOW);
+
+                return null;
+            },
+            IgniteException.class, null
+        );
+    }
+
+    /**
+     * Tests that DATEDIFF SQL function works for a {@link LocalDateTime}
+     * with the time part set to midnight field converted to
+     * a {@link Timestamp} object via a custom SQL function.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDateDiffForLocalDateTimeFieldAtMidnight() throws Exception {
+        SqlFieldsQuery qry = new SqlFieldsQuery(
+            "select DATEDIFF('DAY', localDateTimeToTimestamp(locDateTime), CURRENT_DATE ()) from EntityWithJsr310Fields");
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertTrue((Long)qryResults.get(0).get(0) >= DAYS_BEFORE_NOW);
+    }
+
+    /**
+     * Tests that selection of a {@link LocalTime} field.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSelectLocalTimeField() throws Exception {
+        SqlFieldsQuery qry = new SqlFieldsQuery("select locTime from EntityWithJsr310Fields");
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(LocalTime.class, qryResults.get(0).get(0).getClass());
+    }
+
+    /**
+     * Tests that selection of a {@link LocalDate} field.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSelectLocalDateField() throws Exception {
+        SqlFieldsQuery qry = new SqlFieldsQuery("select locDate from EntityWithJsr310Fields");
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(LocalDate.class, qryResults.get(0).get(0).getClass());
+    }
+
+    /**
+     * Tests that selection of a {@link LocalDateTime} field.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSelectLocalDateTimeField() throws Exception {
+        SqlFieldsQuery qry = new SqlFieldsQuery("select locDateTime from EntityWithJsr310Fields");
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(LocalDateTime.class, qryResults.get(0).get(0).getClass());
+    }
+
+    /**
+     * Tests selection of an entity by a {@link LocalTime} field.
+     */
+    public void testSelectByAllJsr310Fields() {
+        SqlFieldsQuery qry = new SqlFieldsQuery(
+            "select _val from EntityWithJsr310Fields where locTime = ? and locDate = ? and locDateTime = ?"
+        ).setArgs(entity.getLocalTime(), entity.getLocalDate(), entity.getLocalDateTime());
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(entity, qryResults.get(0).get(0));
+    }
+
+    /**
+     * Tests updating of all JSR-310 fields.
+     */
+    public void testUpdateAllJsr310Fields() {
+        EntityWithJsr310Fields expEntity = new EntityWithJsr310Fields(entity);
+
+        expEntity.setLocalTime(expEntity.getLocalTime().plusHours(1));
+        expEntity.setLocalDate(expEntity.getLocalDate().plusDays(1));
+        expEntity.setLocalDateTime(LocalDateTime.of(expEntity.getLocalDate(), expEntity.getLocalTime()));
+
+        SqlFieldsQuery qry = new SqlFieldsQuery(
+            "update EntityWithJsr310Fields set locTime = ?, locDate = ?, locDateTime = ? where id = ?"
+        ).setArgs(expEntity.getLocalTime(), expEntity.getLocalDate(), expEntity.getLocalDateTime(), entity.getId());
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(1L, qryResults.get(0).get(0));
+        assertEquals(expEntity, cache.get(expEntity.getId()));
+    }
+
+    /**
+     * Tests deleting by all JSR-310 fields.
+     */
+    public void testDeleteByAllJsr310Fields() {
+        SqlFieldsQuery qry = new SqlFieldsQuery(
+            "delete from EntityWithJsr310Fields where locTime = ? and locDate = ? and locDateTime = ?"
+        ).setArgs(entity.getLocalTime(), entity.getLocalDate(), entity.getLocalDateTime());
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(1L, qryResults.get(0).get(0));
+        assertEquals(0, cache.size());
+    }
+}
diff --git a/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiBaseTest.java b/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiBaseTest.java
new file mode 100644
index 0000000000000..a65d997e3532d
--- /dev/null
+++ b/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiBaseTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Base class for JSR-310 Java 8 Date and Time API queries tests.
+ */
+public abstract class CacheQueryJsr310Java8DateTimeApiBaseTest extends GridCommonAbstractTest {
+    /** {@link LocalTime} instance. */
+    protected static final LocalTime LOCAL_TIME = LocalTime.now().minusHours(10);
+
+    /**
+     *  The number of days subtracted from the current time when constructing
+     *  {@link LocalDate} and {@link LocalDateTime}
+     *  instances.
+     */
+    protected static final long DAYS_BEFORE_NOW = 10;
+
+    /** {@link LocalDate} instance. */
+    protected static final LocalDate LOCAL_DATE = LocalDate.now().minusDays(DAYS_BEFORE_NOW);
+
+    /** {@link LocalDateTime} instance. */
+    protected static final LocalDateTime LOCAL_DATE_TIME = LocalDateTime.of(LOCAL_DATE, LocalTime.MIDNIGHT);
+
+    /**
+     * Creates a cache configuration with the specified cache name
+     * and indexed type key/value pairs.
+     *
+     * @param cacheName Cache name
+     * @param indexedTypes key/value pairs according to {@link CacheConfiguration#setIndexedTypes(Class[])}.
+     * @param <K> Key type.
+     * @param <V> Value type.
+     * @return Cache configuration.
+     */
+    protected static <K, V> CacheConfiguration<K, V> createCacheConfig(String cacheName, Class<?>... indexedTypes) {
+        return new CacheConfiguration<K, V>(cacheName)
+            .setCacheMode(CacheMode.REPLICATED)
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setIndexedTypes(indexedTypes);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+}
diff --git a/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiKeyAndValueTest.java b/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiKeyAndValueTest.java
new file mode 100644
index 0000000000000..967598edf9425
--- /dev/null
+++ b/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiKeyAndValueTest.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2;
+
+import java.io.Serializable;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheEntry;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.A;
+
+/**
+ * Tests queries against JSR-310 Java 8 Date and Time API keys and values.
+ */
+public class CacheQueryJsr310Java8DateTimeApiKeyAndValueTest extends CacheQueryJsr310Java8DateTimeApiBaseTest {
+    /** Ignite instance. */
+    private Ignite ignite;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        ignite = startGridsMultiThreaded(1, true);
+    }
+
+    /**
+     * Tests that the inserted key-value pair is stored as is.
+     *
+     * @param key Key.
+     * @param val Value.
+     * @param <K> Key type.
+     * @param <V> Value type
+     */
+    private <K, V> void doTestInsertedKeyValuePairStoredAsIs(K key, V val) {
+        A.notNull(key, "key");
+        A.notNull(val, "val");
+
+        final Class<?> keyCls = key.getClass();
+        final Class<?> valCls = val.getClass();
+
+        IgniteCache<K, V> cache =
+            ignite.getOrCreateCache(createCacheConfig(valCls.getSimpleName() + "Cache", keyCls, valCls));
+
+        SqlFieldsQuery qry =
+            new SqlFieldsQuery("insert into " + valCls.getSimpleName() + "(_key, _val) values(?, ?)").setArgs(key, val);
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(1L, qryResults.get(0).get(0));
+        assertEquals(1, cache.size());
+
+        CacheEntry<K, V> entry = cache.getEntry(key);
+        List<Cache.Entry<Object, Object>> all = cache.query(new ScanQuery<>()).getAll();
+
+        assertNotNull(entry);
+        assertEquals(key, entry.getKey());
+        assertEquals(val, entry.getValue());
+        assertEquals(keyCls, entry.getKey().getClass());
+        assertEquals(valCls, entry.getValue().getClass());
+    }
+
+    /**
+     * Tests that selection by a key-value pair returns the expected key and value types.
+     *
+     * @param key Key.
+     * @param val Value.
+     * @param expKeyCls Expected key class.
+     * @param expValCls Expected value class.
+     * @param <K> Key type.
+     * @param <V> Value type.
+     */
+    private <K, V> void doTestSelectByKeyValuePairReturnsExpectedKeyAndValueTypes(
+        K key, V val, Class<?> expKeyCls, Class<?> expValCls
+    ) {
+        A.notNull(key, "key");
+        A.notNull(val, "val");
+
+        final Class<?> keyCls = key.getClass();
+        final Class<?> valCls = val.getClass();
+
+        IgniteCache<K, V> cache =
+            ignite.getOrCreateCache(createCacheConfig(valCls.getSimpleName() + "Cache", keyCls, valCls));
+
+        cache.put(key, val);
+
+        SqlFieldsQuery qry =
+            new SqlFieldsQuery("select _key, _val from " + valCls.getSimpleName() + " where _key=? and _val=?")
+                .setArgs(key, val);
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(expKeyCls, qryResults.get(0).get(0).getClass());
+        assertEquals(expValCls, qryResults.get(0).get(1).getClass());
+    }
+
+    /**
+     * Tests updating a value by a key-value pair.
+     *
+     * @param key Key.
+     * @param val Value.
+     * @param newVal New value.
+     * @param <K> Key type.
+     * @param <V> Value type.
+     */
+    private <K, V> void doTestUpdateValueByKeyValuePair(K key, V val, V newVal) {
+        A.notNull(key, "key");
+        A.notNull(val, "val");
+        A.notNull(newVal, "newVal");
+
+        final Class<?> keyCls = key.getClass();
+        final Class<?> valCls = val.getClass();
+
+        IgniteCache<K, V> cache =
+            ignite.getOrCreateCache(createCacheConfig(valCls.getSimpleName() + "Cache", keyCls, valCls));
+
+        cache.put(key, val);
+
+        SqlFieldsQuery qry =
+            new SqlFieldsQuery("update " + valCls.getSimpleName() + " set _val=? where _key=? and _val=?")
+                .setArgs(newVal, key, val);
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(1L, qryResults.get(0).get(0));
+        assertEquals(newVal, cache.get(key));
+    }
+
+    /**
+     * Tests deleting by a key-value pair.
+     *
+     * @param key Key.
+     * @param val Value.
+     * @param <K> Key type.
+     * @param <V> Value type.
+     */
+    private <K, V> void doTestDeleteByKeyValuePair(K key, V val) {
+        A.notNull(key, "key");
+        A.notNull(val, "val");
+
+        final Class<?> keyCls = key.getClass();
+        final Class<?> valCls = val.getClass();
+
+        IgniteCache<K, V> cache =
+            ignite.getOrCreateCache(createCacheConfig(valCls.getSimpleName() + "Cache", keyCls, valCls));
+
+        cache.put(key, val);
+
+        SqlFieldsQuery qry =new SqlFieldsQuery("delete from " + valCls.getSimpleName() + " where _key=? and _val=?")
+            .setArgs(key, val);
+
+        List<List<?>> qryResults = cache.query(qry).getAll();
+
+        assertEquals(1, qryResults.size());
+        assertEquals(1L, qryResults.get(0).get(0));
+        assertEquals(0, cache.size());
+    }
+
+    /**
+     * Tests that the inserted {@link LocalTime} key-value pair is stored as is.
+     *
+     * @throws Exception If failed.
+     */
+    public void testInsertedLocalTimeKeyValuePairStoredAsIs() throws Exception {
+        doTestInsertedKeyValuePairStoredAsIs(LOCAL_TIME, LOCAL_TIME);
+    }
+
+    /**
+     * Tests that the inserted {@link LocalDate} key-value pair is stored as is.
+     *
+     * @throws Exception If failed.
+     */
+    public void testInsertedLocalDateKeyValuePairStoredAsIs() throws Exception {
+        doTestInsertedKeyValuePairStoredAsIs(LOCAL_DATE, LOCAL_DATE);
+    }
+
+    /**
+     * Tests that the inserted {@link LocalDateTime} key-value pair is stored as is.
+     *
+     * @throws Exception If failed.
+     */
+    public void testInsertedLocalDateTimeKeyValuePairStoredAsIs() throws Exception {
+        doTestInsertedKeyValuePairStoredAsIs(LOCAL_DATE_TIME, LOCAL_DATE_TIME);
+    }
+
+    /**
+     * Tests that selection by a {@link LocalTime} key-value pair returns {@link LocalTime} key and value types.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSelectByLocalTimeKeyValuePairReturnsLocalTimeKeyAndValue() throws Exception {
+        doTestSelectByKeyValuePairReturnsExpectedKeyAndValueTypes(
+            LOCAL_TIME, LOCAL_TIME, LocalTime.class, LocalTime.class
+        );
+    }
+
+    /**
+     * Tests that selection by a {@link LocalDate} key-value pair returns {@link LocalDate} key and value types.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSelectByLocalDateKeyValuePairReturnsLocalDateKeyAndValue() throws Exception {
+        doTestSelectByKeyValuePairReturnsExpectedKeyAndValueTypes(
+            LOCAL_DATE, LOCAL_DATE, LocalDate.class, LocalDate.class
+        );
+    }
+
+    /**
+     * Tests that selection by a {@link LocalDateTime} key-value pair returns {@link LocalDateTime} key and value types.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSelectByLocalDateTimeKeyValuePairReturnsLocalDateTimeKeyAndValue() throws Exception {
+        doTestSelectByKeyValuePairReturnsExpectedKeyAndValueTypes(
+            LOCAL_DATE_TIME, LOCAL_DATE_TIME, LocalDateTime.class, LocalDateTime.class
+        );
+    }
+
+    /**
+     * Tests updating a value by a {@link LocalTime} key-value pair.
+     *
+     * @throws Exception If failed.
+     */
+    public void testUpdateValueByLocalTimeKeyValuePair() throws Exception {
+        doTestUpdateValueByKeyValuePair(LOCAL_TIME, LOCAL_TIME, LOCAL_TIME.plusHours(1));
+    }
+
+    /**
+     * Tests updating a value by a {@link LocalDate} key-value pair.
+     *
+     * @throws Exception If failed.
+     */
+    public void testUpdateValueByLocalDateKeyValuePair() throws Exception {
+        doTestUpdateValueByKeyValuePair(LOCAL_DATE, LOCAL_DATE, LOCAL_DATE.plusDays(1));
+    }
+
+    /**
+     * Tests updating a value by a {@link LocalDateTime} key-value pair.
+     *
+     * @throws Exception If failed.
+     */
+    public void testUpdateValueByLocalDateTimeKeyValuePair() throws Exception {
+        doTestUpdateValueByKeyValuePair(LOCAL_DATE_TIME, LOCAL_DATE_TIME, LOCAL_DATE_TIME.plusHours(1));
+    }
+
+    /**
+     * Test deleting by a {@link LocalTime} key-value pair.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDeleteByLocalTimeKeyValuePair() throws Exception {
+        doTestDeleteByKeyValuePair(LOCAL_TIME, LOCAL_TIME);
+    }
+
+    /**
+     * Test deleting by a {@link LocalDate} key-value pair.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDeleteByLocalDateKeyValuePair() throws Exception {
+        doTestDeleteByKeyValuePair(LOCAL_DATE, LOCAL_DATE);
+    }
+
+    /**
+     * Test deleting by a {@link LocalDateTime} key-value pair.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDeleteByLocalDateTimeKeyValuePair() throws Exception {
+        doTestDeleteByKeyValuePair(LOCAL_DATE_TIME, LOCAL_DATE_TIME);
+    }
+}
diff --git a/modules/indexing/src/test/java8/org/apache/ignite/testsuites/CacheQueryJsr310Java8DateTimeApiSupportTestSuite.java b/modules/indexing/src/test/java8/org/apache/ignite/testsuites/CacheQueryJsr310Java8DateTimeApiSupportTestSuite.java
new file mode 100644
index 0000000000000..4c018368e4aa9
--- /dev/null
+++ b/modules/indexing/src/test/java8/org/apache/ignite/testsuites/CacheQueryJsr310Java8DateTimeApiSupportTestSuite.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.query.h2.CacheQueryEntityWithJsr310Java8DateTimeApiFieldsTest;
+import org.apache.ignite.internal.processors.query.h2.CacheQueryJsr310Java8DateTimeApiKeyAndValueTest;
+
+/**
+ * Test suite for JSR-310 Java 8 Date and Time API queries.
+ */
+public class CacheQueryJsr310Java8DateTimeApiSupportTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception If failed.
+     */
+    public static TestSuite suite() throws Exception {
+        TestSuite suite = new TestSuite("JSR-310 Java 8 Date and Time API Cache Queries Test Suite");
+
+        suite.addTestSuite(CacheQueryEntityWithJsr310Java8DateTimeApiFieldsTest.class);
+        suite.addTestSuite(CacheQueryJsr310Java8DateTimeApiKeyAndValueTest.class);
+
+        return suite;
+    }
+}
diff --git a/parent/pom.xml b/parent/pom.xml
index 59afbf20ad771..391415a957805 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -116,6 +116,10 @@
         <osgi.embed.transitive>false</osgi.embed.transitive>
         <osgi.fail.ok>false</osgi.fail.ok>
 
+
+        <!-- will be changed by profile activation. allows to combine profiles. -->
+        <java8.folder>${project.build.sourceDirectory}</java8.folder>
+        <java8.test.folder>${project.build.testSourceDirectory}</java8.test.folder>
     </properties>
 
     <groupId>org.apache.ignite</groupId>
@@ -546,6 +550,11 @@
                         </execution>
                     </executions>
                 </plugin>
+                <plugin>
+                    <groupId>org.codehaus.mojo</groupId>
+                    <artifactId>build-helper-maven-plugin</artifactId>
+                    <version>3.0.0</version>
+                </plugin>
             </plugins>
         </pluginManagement>
 
@@ -559,6 +568,37 @@
                 </configuration>
             </plugin>
 
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>add-sources</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${java8.folder}</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>add-tests</id>
+                        <phase>generate-test-sources</phase>
+                        <goals>
+                            <goal>add-test-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${java8.test.folder}</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
@@ -938,6 +978,8 @@
             </activation>
             <properties>
                 <javadoc.opts>-Xdoclint:none</javadoc.opts>
+                <java8.folder>${project.build.sourceDirectory}/../java8</java8.folder>
+                <java8.test.folder>${project.build.testSourceDirectory}/../java8</java8.test.folder>
             </properties>
             <build>
                 <plugins>

From 4ee9e7da3187245fb34d87e2475b69bb5b474b0c Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Wed, 16 Aug 2017 18:00:31 +0300
Subject: [PATCH 320/516] gg-12637 Fixed unevenly partitions distribution in
 FairAffinityFunction

---
 .../affinity/fair/FairAffinityFunction.java   | 29 ++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
index 7acb5b429e693..fe146ea7ad191 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
@@ -95,6 +95,15 @@ public class FairAffinityFunction implements AffinityFunction {
     /** Exclude neighbors warning. */
     private transient boolean exclNeighborsWarn;
 
+    /**
+     * <b>NOTE:</b> Use {@code true} value only for new clusters or in case of cluster upgrade with downtime.
+     * If nodes of cluster have affinity function with different configuration it will lead to different
+     * assignments calculation on different nodes and, therefore, broken consistent hashing functionality.
+     *
+     * Compatibility flag. Lead to better partitions distribution if value is {@code true}.
+     */
+    private boolean ceilIdealPartCnt = false;
+
     /** Logger instance. */
     @LoggerResource
     private transient IgniteLogger log;
@@ -280,6 +289,24 @@ public void setExcludeNeighbors(boolean exclNeighbors) {
         this.exclNeighbors = exclNeighbors;
     }
 
+    /**
+     * Returns value of {@link #ceilIdealPartCnt} compatibility flag.
+     *
+     * @return Value of {@link #ceilIdealPartCnt} compatibility flag.
+     */
+    public boolean isCeilIdealPartitionsCount() {
+        return ceilIdealPartCnt;
+    }
+
+    /**
+     * Sets value of {@link #ceilIdealPartCnt} compatibility flag.
+     *
+     * @param ceilIdealPartCnt Indicates that ideal partitions count should be rounded to biggest integer value.
+     */
+    public void setCeilIdealPartitionsCount(boolean ceilIdealPartCnt) {
+        this.ceilIdealPartCnt = ceilIdealPartCnt;
+    }
+
     /** {@inheritDoc} */
     @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext ctx) {
         List<ClusterNode> topSnapshot = ctx.currentTopologySnapshot();
@@ -386,7 +413,7 @@ private void assignPending(int tier,
         if (F.isEmpty(pending))
             return;
 
-        int idealPartCnt = parts / topSnapshot.size();
+        int idealPartCnt = ceilIdealPartCnt ? (int)Math.ceil((double)parts / topSnapshot.size()) : parts / topSnapshot.size();
 
         Map<UUID, PartitionSet> tierMapping = fullMap.tierMapping(tier);
 

From f9c5bb0a00877df508c407f8daf0c350fd1bc99f Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 9 Aug 2017 18:58:02 +0700
Subject: [PATCH 321/516] IGNITE-5987 Added -nq (visor will not quit in batch
 mode) option for Visor Cmd. (cherry picked from commit 9b9eabd)

---
 .../apache/ignite/visor/commands/VisorConsole.scala  | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
index ad8c2edaaf8dd..31b4ff06b00dc 100644
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
@@ -93,6 +93,7 @@ class VisorConsole {
             println("        -cfg=<path>          - connect with specified configuration.")
             println("        -b=<path>            - batch mode with file.")
             println("        -e=cmd1;cmd2;...     - batch mode with commands.")
+            println("        -nq                  - batch mode will not quit after execution (useful for alerts monitoring).")
 
             visor.quit()
         }
@@ -104,6 +105,10 @@ class VisorConsole {
         val cfgFile = argValue("cfg", argLst)
         val batchFile = argValue("b", argLst)
         val batchCommand = argValue("e", argLst)
+        val noBatchQuit = hasArgName("nq", argLst)
+
+        if (noBatchQuit && batchFile.isEmpty && batchCommand.isEmpty)
+            visor.warn("Option \"-nq\" will be ignored because batch mode options \"-b\" or \"-e\" were not specified.")
 
         cfgFile.foreach(cfg => {
             if (cfg.trim.isEmpty) {
@@ -150,7 +155,10 @@ class VisorConsole {
             case Some(cmd) =>
                 visor.batchMode = true
 
-                new ByteArrayInputStream((cmd + "\nquit\n").getBytes("UTF-8"))
+                val script = if (noBatchQuit) cmd else cmd + "\nquit\n"
+
+                new ByteArrayInputStream(script.getBytes("UTF-8"))
+
             case None => new FileInputStream(FileDescriptor.in)
         }
 
@@ -160,7 +168,7 @@ class VisorConsole {
 
             new TerminalSupport(false) {}
         } catch {
-            case ignored: ClassNotFoundException => null
+            case _: ClassNotFoundException => null
         }
 
         val reader = new ConsoleReader(inputStream, System.out, term)

From a76932f6f3d253a292803a0d1e954d42589bdbc6 Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Wed, 16 Aug 2017 18:00:31 +0300
Subject: [PATCH 322/516] gg-12637 Fixed unevenly partitions distribution in
 FairAffinityFunction

---
 .../affinity/fair/FairAffinityFunction.java   | 29 ++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
index cffcf108c18df..522dac2144db1 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
@@ -95,6 +95,15 @@ public class FairAffinityFunction implements AffinityFunction {
     /** Exclude neighbors warning. */
     private transient boolean exclNeighborsWarn;
 
+    /**
+     * <b>NOTE:</b> Use {@code true} value only for new clusters or in case of cluster upgrade with downtime.
+     * If nodes of cluster have affinity function with different configuration it will lead to different
+     * assignments calculation on different nodes and, therefore, broken consistent hashing functionality.
+     *
+     * Compatibility flag. Lead to better partitions distribution if value is {@code true}.
+     */
+    private boolean ceilIdealPartCnt = false;
+
     /** Logger instance. */
     @LoggerResource
     private transient IgniteLogger log;
@@ -280,6 +289,24 @@ public void setExcludeNeighbors(boolean exclNeighbors) {
         this.exclNeighbors = exclNeighbors;
     }
 
+    /**
+     * Returns value of {@link #ceilIdealPartCnt} compatibility flag.
+     *
+     * @return Value of {@link #ceilIdealPartCnt} compatibility flag.
+     */
+    public boolean isCeilIdealPartitionsCount() {
+        return ceilIdealPartCnt;
+    }
+
+    /**
+     * Sets value of {@link #ceilIdealPartCnt} compatibility flag.
+     *
+     * @param ceilIdealPartCnt Indicates that ideal partitions count should be rounded to biggest integer value.
+     */
+    public void setCeilIdealPartitionsCount(boolean ceilIdealPartCnt) {
+        this.ceilIdealPartCnt = ceilIdealPartCnt;
+    }
+
     /** {@inheritDoc} */
     @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext ctx) {
         List<ClusterNode> topSnapshot = ctx.currentTopologySnapshot();
@@ -382,7 +409,7 @@ private void assignPending(int tier,
         if (F.isEmpty(pending))
             return;
 
-        int idealPartCnt = parts / topSnapshot.size();
+        int idealPartCnt = ceilIdealPartCnt ? (int)Math.ceil((double)parts / topSnapshot.size()) : parts / topSnapshot.size();
 
         Map<UUID, PartitionSet> tierMapping = fullMap.tierMapping(tier);
 

From 3e7849fbdfc60bfe6d24ab734672924293394f8b Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Wed, 16 Aug 2017 18:00:31 +0300
Subject: [PATCH 323/516] gg-12637 Fixed unevenly partitions distribution in
 FairAffinityFunction

---
 .../affinity/fair/FairAffinityFunction.java   | 29 ++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
index 7acb5b429e693..fe146ea7ad191 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
@@ -95,6 +95,15 @@ public class FairAffinityFunction implements AffinityFunction {
     /** Exclude neighbors warning. */
     private transient boolean exclNeighborsWarn;
 
+    /**
+     * <b>NOTE:</b> Use {@code true} value only for new clusters or in case of cluster upgrade with downtime.
+     * If nodes of cluster have affinity function with different configuration it will lead to different
+     * assignments calculation on different nodes and, therefore, broken consistent hashing functionality.
+     *
+     * Compatibility flag. Lead to better partitions distribution if value is {@code true}.
+     */
+    private boolean ceilIdealPartCnt = false;
+
     /** Logger instance. */
     @LoggerResource
     private transient IgniteLogger log;
@@ -280,6 +289,24 @@ public void setExcludeNeighbors(boolean exclNeighbors) {
         this.exclNeighbors = exclNeighbors;
     }
 
+    /**
+     * Returns value of {@link #ceilIdealPartCnt} compatibility flag.
+     *
+     * @return Value of {@link #ceilIdealPartCnt} compatibility flag.
+     */
+    public boolean isCeilIdealPartitionsCount() {
+        return ceilIdealPartCnt;
+    }
+
+    /**
+     * Sets value of {@link #ceilIdealPartCnt} compatibility flag.
+     *
+     * @param ceilIdealPartCnt Indicates that ideal partitions count should be rounded to biggest integer value.
+     */
+    public void setCeilIdealPartitionsCount(boolean ceilIdealPartCnt) {
+        this.ceilIdealPartCnt = ceilIdealPartCnt;
+    }
+
     /** {@inheritDoc} */
     @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext ctx) {
         List<ClusterNode> topSnapshot = ctx.currentTopologySnapshot();
@@ -386,7 +413,7 @@ private void assignPending(int tier,
         if (F.isEmpty(pending))
             return;
 
-        int idealPartCnt = parts / topSnapshot.size();
+        int idealPartCnt = ceilIdealPartCnt ? (int)Math.ceil((double)parts / topSnapshot.size()) : parts / topSnapshot.size();
 
         Map<UUID, PartitionSet> tierMapping = fullMap.tierMapping(tier);
 

From a1e51e652665d0c1d368bf73976b2b62bd3cb23d Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <e.zhuravlev.wk@gmail.com>
Date: Thu, 17 Aug 2017 16:15:31 +0300
Subject: [PATCH 324/516] IGNITE-6088 Socket#shutdownOutput in ServerImpl leads
 to UnsupportedOperationException on SSLSocket.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 4574b3816af8f..22cb618623644 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -57,6 +57,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLSocket;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -5804,7 +5805,8 @@ private class SocketReader extends IgniteSpiThread {
 
                             spi.writeToSocket(sock, res, timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()));
 
-                            sock.shutdownOutput();
+                            if (!(sock instanceof SSLSocket))
+                                sock.shutdownOutput();
 
                             if (log.isInfoEnabled())
                                 log.info("Finished writing ping response " + "[rmtNodeId=" + msg.creatorNodeId() +

From b1e0bdb5186dee6ebf65054b30674e5d33247fc1 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Wed, 16 Aug 2017 00:24:07 +0300
Subject: [PATCH 325/516] IgniteCacheNearRestartRollbackSelfTest#testRestarts
 is muted. (cherry picked from commit 60e2de7)

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../distributed/IgniteCacheNearRestartRollbackSelfTest.java     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index 3f242b5726867..aea4d7782d858 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -132,6 +132,8 @@ protected CacheConfiguration<Object, Object> cacheConfiguration(String gridName)
      */
     @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
     public void testRestarts() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-12398");
+
         startGrids(4);
 
         Ignite tester = ignite(3);

From d5c8e5f0ade84dafbd81d8592bbab08c099782b5 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Wed, 16 Aug 2017 14:06:37 +0300
Subject: [PATCH 326/516] Fixed flaky test "IgniteCacheEntryListener*" (cherry
 picked from commit 928d445)

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../query/continuous/CacheContinuousQueryEntry.java      | 9 +--------
 .../query/continuous/CacheContinuousQueryHandler.java    | 2 --
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index 366a1e05fa46c..ffbbc2c1e05ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -199,13 +199,6 @@ void markFiltered() {
         depInfo = null;
     }
 
-    /**
-     * @param topVer Topology version.
-     */
-    void topologyVersion(AffinityTopologyVersion topVer) {
-        this.topVer = topVer;
-    }
-
     /**
      * @return Size include this event and filtered.
      */
@@ -222,7 +215,7 @@ CacheContinuousQueryEntry forBackupQueue() {
             return this;
 
         CacheContinuousQueryEntry e =
-            new CacheContinuousQueryEntry(cacheId, null, null, null, null, keepBinary, part, updateCntr, null);
+            new CacheContinuousQueryEntry(cacheId, null, null, null, null, keepBinary, part, updateCntr, topVer);
 
         e.flags = flags;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 299a95f720e28..0dbf1f402240d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -481,8 +481,6 @@ public void keepBinary(boolean keepBinary) {
                             for (CacheContinuousQueryEntry e : backupQueue0) {
                                 if (!e.isFiltered())
                                     prepareEntry(cctx, nodeId, e);
-
-                                e.topologyVersion(topVer);
                             }
 
                             ctx.continuous().addBackupNotification(nodeId, routineId, backupQueue0, topic);

From 15de7d4fda1e8ccd4102c82851973d3ed3e47a2e Mon Sep 17 00:00:00 2001
From: Nikolay Izhikov <nizhikov.dev@gmail.com>
Date: Wed, 16 Aug 2017 15:45:16 +0300
Subject: [PATCH 327/516] IGNITE-5897 Fix session init/end logic. This fixes
 tests.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../cache/store/CacheStoreManager.java        |  4 +++-
 .../store/GridCacheStoreManagerAdapter.java   | 20 +++++++++----------
 .../cache/transactions/IgniteTxAdapter.java   |  9 +++++++--
 3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
index 8d6b63dce3c8a..b096edf6a8b95 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
@@ -165,9 +165,11 @@ public boolean removeAll(@Nullable IgniteInternalTx tx, Collection<Object> keys)
     /**
      * @param tx Transaction.
      * @param commit Commit.
+     * @param last {@code True} if this is last store in transaction.
+     * @param storeSessionEnded {@code True} if session for underlying store already ended.
      * @throws IgniteCheckedException If failed.
      */
-    public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException;
+    public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last, boolean storeSessionEnded) throws IgniteCheckedException;
 
     /**
      * End session initiated by write-behind store.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index cebc5f1f62077..0753af4f1d2a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -787,7 +787,8 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public final void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException {
+    @Override public final void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last,
+        boolean storeSessionEnded) throws IgniteCheckedException {
         assert store != null;
 
         sessionInit0(tx);
@@ -798,7 +799,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
                     lsnr.onSessionEnd(locSes, commit);
             }
 
-            if (!sesHolder.get().ended(store))
+            if (!sesHolder.get().ended(store) && !storeSessionEnded)
                 store.sessionEnd(commit);
         }
         catch (Throwable e) {
@@ -867,7 +868,7 @@ private void sessionInit0(@Nullable IgniteInternalTx tx) throws IgniteCheckedExc
         sesHolder.set(ses);
 
         try {
-            if (sesLsnrs != null && !ses.started(this)) {
+            if (!ses.started(store) && sesLsnrs != null) {
                 for (CacheStoreSessionListener lsnr : sesLsnrs)
                     lsnr.onSessionStart(locSes);
             }
@@ -930,11 +931,8 @@ private static class SessionData {
         private Object attachment;
 
         /** */
-        private final Set<CacheStoreManager> started =
-            new GridSetWrapper<>(new IdentityHashMap<CacheStoreManager, Object>());
-
-        /** */
-        private final Set<CacheStore> ended = new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
+        private final Set<CacheStore> started =
+            new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
 
         /**
          * @param tx Current transaction.
@@ -997,8 +995,8 @@ private void cacheName(String cacheName) {
         /**
          * @return If session is started.
          */
-        private boolean started(CacheStoreManager mgr) {
-            return !started.add(mgr);
+        private boolean started(CacheStore store) {
+            return !started.add(store);
         }
 
         /**
@@ -1006,7 +1004,7 @@ private boolean started(CacheStoreManager mgr) {
          * @return Whether session already ended on this store instance.
          */
         private boolean ended(CacheStore store) {
-            return !ended.add(store);
+            return !started.remove(store);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index b07a1175c436d..266c5a83cd8c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -40,6 +41,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -61,6 +63,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
+import org.apache.ignite.internal.util.GridSetWrapper;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
 import org.apache.ignite.internal.util.lang.GridTuple;
@@ -1238,13 +1241,15 @@ protected boolean isWriteToStoreFromDhtValid(Collection<CacheStoreManager> store
      * @param commit Commit flag.
      * @throws IgniteCheckedException In case of error.
      */
-    protected void sessionEnd(Collection<CacheStoreManager> stores, boolean commit) throws IgniteCheckedException {
+    protected void sessionEnd(final Collection<CacheStoreManager> stores, boolean commit) throws IgniteCheckedException {
         Iterator<CacheStoreManager> it = stores.iterator();
 
+        Set<CacheStore> visited = new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
+
         while (it.hasNext()) {
             CacheStoreManager store = it.next();
 
-            store.sessionEnd(this, commit, !it.hasNext());
+            store.sessionEnd(this, commit, !it.hasNext(), !visited.add(store.store()));
         }
     }
 

From 2627c1eb8bb91b90b2f0899c30a28ea98ac562b9 Mon Sep 17 00:00:00 2001
From: Alexander Fedotov <alexander.fedotoff@gmail.com>
Date: Fri, 18 Aug 2017 16:00:30 +0300
Subject: [PATCH 328/516] IGNITE-GG-12646 Fix tests for JSR-310 Java 8 Date and
 Time API introduced with IGNITE-GG-12549

---
 ...CacheQueryJsr310Java8DateTimeApiBaseTest.java | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiBaseTest.java b/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiBaseTest.java
index a65d997e3532d..4e86c6d2c46bf 100644
--- a/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiBaseTest.java
+++ b/modules/indexing/src/test/java8/org/apache/ignite/internal/processors/query/h2/CacheQueryJsr310Java8DateTimeApiBaseTest.java
@@ -24,12 +24,19 @@
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
  * Base class for JSR-310 Java 8 Date and Time API queries tests.
  */
 public abstract class CacheQueryJsr310Java8DateTimeApiBaseTest extends GridCommonAbstractTest {
+    /** IP finder. */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
     /** {@link LocalTime} instance. */
     protected static final LocalTime LOCAL_TIME = LocalTime.now().minusHours(10);
 
@@ -46,6 +53,15 @@ public abstract class CacheQueryJsr310Java8DateTimeApiBaseTest extends GridCommo
     /** {@link LocalDateTime} instance. */
     protected static final LocalDateTime LOCAL_DATE_TIME = LocalDateTime.of(LOCAL_DATE, LocalTime.MIDNIGHT);
 
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+        TcpDiscoverySpi discoverySpi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+
+        discoverySpi.setIpFinder(IP_FINDER);
+
+        return cfg;
+    }
+
     /**
      * Creates a cache configuration with the specified cache name
      * and indexed type key/value pairs.

From 533128821357c0909710069ea589894d99908474 Mon Sep 17 00:00:00 2001
From: Vyacheslav Daradur <daradurvs@gmail.com>
Date: Fri, 10 Feb 2017 16:51:37 +0300
Subject: [PATCH 329/516] GG-12647: Backport IGNITE-3196 Add support for
 BigDecimals with negative scale in BinaryMarshaller

---
 .../ignite/internal/binary/BinaryUtils.java   | 12 ++---
 .../internal/binary/BinaryWriterExImpl.java   | 14 +++---
 .../binary/BinaryMarshallerSelfTest.java      | 30 +++++++++++++
 modules/platforms/cpp/odbc/src/utility.cpp    | 17 ++++---
 .../Compute/ComputeApiTest.cs                 | 18 ++++----
 .../Impl/Binary/BinaryUtils.cs                | 44 +++++++++++--------
 6 files changed, 92 insertions(+), 43 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index fdc54c7871711..ceadd2a59585c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -65,8 +65,8 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 
 /**
  * Binary utils.
@@ -1193,13 +1193,15 @@ public static BigDecimal doReadDecimal(BinaryInputStream in) {
         int scale = in.readInt();
         byte[] mag = doReadByteArray(in);
 
-        BigInteger intVal = new BigInteger(mag);
+        boolean negative = mag[0] < 0;
 
-        if (scale < 0) {
-            scale &= 0x7FFFFFFF;
+        if (negative)
+            mag[0] &= 0x7F;
 
+        BigInteger intVal = new BigInteger(mag);
+
+        if (negative)
             intVal = intVal.negate();
-        }
 
         return new BigDecimal(intVal, scale);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index 1de0a6533e600..12ce868354afc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -364,18 +364,20 @@ public void doWriteDecimal(@Nullable BigDecimal val) {
 
             out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL);
 
+            out.unsafeWriteInt(val.scale());
+
             BigInteger intVal = val.unscaledValue();
 
-            if (intVal.signum() == -1) {
-                intVal = intVal.negate();
+            boolean negative = intVal.signum() == -1;
 
-                out.unsafeWriteInt(val.scale() | 0x80000000);
-            }
-            else
-                out.unsafeWriteInt(val.scale());
+            if (negative)
+                intVal = intVal.negate();
 
             byte[] vals = intVal.toByteArray();
 
+            if (negative)
+                vals[0] |= -0x80;
+
             out.unsafeWriteInt(vals.length);
             out.writeByteArray(vals);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index 6d07c9ba42770..bfc8bd357f292 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -29,6 +29,7 @@
 import java.lang.reflect.Proxy;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.math.RoundingMode;
 import java.net.InetSocketAddress;
 import java.sql.Timestamp;
 import java.util.AbstractQueue;
@@ -178,6 +179,35 @@ public void testDecimal() throws Exception {
         assertEquals((val = new BigDecimal(new BigInteger("-79228162514264337593543950336"))), marshalUnmarshal(val));
     }
 
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -1)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -2)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -3)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -4)), marshalUnmarshal(val));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleRoundingModeDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.ZERO.setScale(-1, RoundingMode.HALF_UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE).setScale(-3, RoundingMode.HALF_DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE).setScale(-5, RoundingMode.HALF_EVEN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MAX_VALUE).setScale(-8, RoundingMode.UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MIN_VALUE).setScale(-10, RoundingMode.DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MAX_VALUE).setScale(-12, RoundingMode.CEILING)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MIN_VALUE).setScale(-15, RoundingMode.FLOOR)), marshalUnmarshal(val));
+    }
+
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/platforms/cpp/odbc/src/utility.cpp b/modules/platforms/cpp/odbc/src/utility.cpp
index 19be799a10aae..c331332a34756 100644
--- a/modules/platforms/cpp/odbc/src/utility.cpp
+++ b/modules/platforms/cpp/odbc/src/utility.cpp
@@ -93,8 +93,14 @@ namespace ignite
 
             impl::binary::BinaryUtils::ReadInt8Array(reader.GetStream(), mag.data(), static_cast<int32_t>(mag.size()));
 
-            int32_t sign = (scale & 0x80000000) ? -1 : 1;
-            scale = scale & 0x7FFFFFFF;
+            int32_t sign = 1;
+            
+            if (mag[0] < 0)
+            {
+                mag[0] &= 0x7F;
+
+                sign = -1;
+            }
 
             common::Decimal res(mag.data(), static_cast<int32_t>(mag.size()), scale, sign);
 
@@ -107,14 +113,15 @@ namespace ignite
 
             const common::BigInteger &unscaled = decimal.GetUnscaledValue();
 
-            int32_t signFlag = unscaled.GetSign() == -1 ? 0x80000000 : 0;
-
-            writer.WriteInt32(decimal.GetScale() | signFlag);
+            writer.WriteInt32(decimal.GetScale());
 
             common::FixedSizeArray<int8_t> magnitude;
 
             unscaled.MagnitudeToBytes(magnitude);
 
+            if (unscaled.GetSign() == -1)
+                magnitude[0] |= -0x80;
+
             writer.WriteInt32(magnitude.GetSize());
 
             impl::binary::BinaryUtils::WriteInt8Array(writer.GetStream(), magnitude.GetData(), magnitude.GetSize());
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
index 1e0287fe56e8f..15d8dba7895be 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
@@ -818,16 +818,16 @@ public void TestEchoDecimal()
             Assert.AreEqual(val = decimal.Parse("-11,12"), _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { val, val.ToString() }));
 
             // Test echo with overflow.
-            try
-            {
-                _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { null, decimal.MaxValue.ToString() + 1 });
+            var ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, decimal.MaxValue.ToString() + 1}));
 
-                Assert.Fail();
-            }
-            catch (IgniteException)
-            {
-                // No-op.
-            }
+            Assert.AreEqual("Decimal magnitude overflow (must be less than 96 bits): 104", ex.Message);
+
+            // Negative scale. 1E+1 parses to "1 scale -1" on Java side.
+            ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, "1E+1"}));
+
+            Assert.AreEqual("Decimal value scale overflow (must be between 0 and 28): -1", ex.Message);
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
index cc5d8a109ce86..5ce77d0ee4347 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
@@ -893,7 +893,9 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
             // Write scale and negative flag.
             int scale = (vals[3] & 0x00FF0000) >> 16; 
 
-            stream.WriteInt(((vals[3] & 0x80000000) == 0x80000000) ? (int)((uint)scale | 0x80000000) : scale);
+            stream.WriteInt(scale);
+
+            Boolean neg = vals[3] < 0;
 
             if (idx == -1)
             {
@@ -923,13 +925,15 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             if ((part24 & 0x80) == 0x80)
                             {
                                 stream.WriteInt(len + 1);
+                                
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
 
-                                stream.WriteByte(ByteZero);
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len);
 
-                            stream.WriteByte((byte)part24);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part24 | -0x80) : part24));
                             stream.WriteByte((byte)part16);
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
@@ -940,12 +944,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 1);
-
-                            stream.WriteByte((byte)part16);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part16 | -0x80) : part16));
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
                         }
@@ -955,12 +961,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 1);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 2);
-
-                            stream.WriteByte((byte)part8);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part8 | -0x80) : part8));
                             stream.WriteByte((byte)part0);
                         }
                         else
@@ -969,12 +977,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 2);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 3);
 
-                            stream.WriteByte((byte)part0);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part0 | -0x80) : part0));
                         }
                     }
                     else
@@ -997,18 +1007,16 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
         {
             int scale = stream.ReadInt();
 
-            bool neg;
+            bool neg = false;
+
+            byte[] mag = ReadByteArray(stream);
 
-            if (scale < 0)
+            if ((sbyte)mag[0] < 0)
             {
-                scale = scale & 0x7FFFFFFF;
+                mag[0] &= 0x7F;
 
                 neg = true;
             }
-            else
-                neg = false;
-
-            byte[] mag = ReadByteArray(stream);
 
             if (scale < 0 || scale > 28)
                 throw new BinaryObjectException("Decimal value scale overflow (must be between 0 and 28): " + scale);

From 558d755a2f5a8499f0dd690834fd97137245fc63 Mon Sep 17 00:00:00 2001
From: Vyacheslav Daradur <daradurvs@gmail.com>
Date: Fri, 10 Feb 2017 16:51:37 +0300
Subject: [PATCH 330/516] GG-12647: Backport IGNITE-3196 Add support for
 BigDecimals with negative scale in BinaryMarshaller

---
 .../ignite/internal/binary/BinaryUtils.java   | 12 ++---
 .../internal/binary/BinaryWriterExImpl.java   | 14 +++---
 .../binary/BinaryMarshallerSelfTest.java      | 30 +++++++++++++
 modules/platforms/cpp/odbc/src/utility.cpp    | 17 ++++---
 .../Compute/ComputeApiTest.cs                 | 18 ++++----
 .../Impl/Binary/BinaryUtils.cs                | 44 +++++++++++--------
 6 files changed, 92 insertions(+), 43 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 1153d155c0400..6831ef9bb0d66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -65,8 +65,8 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 
 /**
  * Binary utils.
@@ -1193,13 +1193,15 @@ public static BigDecimal doReadDecimal(BinaryInputStream in) {
         int scale = in.readInt();
         byte[] mag = doReadByteArray(in);
 
-        BigInteger intVal = new BigInteger(mag);
+        boolean negative = mag[0] < 0;
 
-        if (scale < 0) {
-            scale &= 0x7FFFFFFF;
+        if (negative)
+            mag[0] &= 0x7F;
 
+        BigInteger intVal = new BigInteger(mag);
+
+        if (negative)
             intVal = intVal.negate();
-        }
 
         return new BigDecimal(intVal, scale);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index adaacdda40676..3289780e3fd0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -403,18 +403,20 @@ public void doWriteDecimal(@Nullable BigDecimal val) {
 
             out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL);
 
+            out.unsafeWriteInt(val.scale());
+
             BigInteger intVal = val.unscaledValue();
 
-            if (intVal.signum() == -1) {
-                intVal = intVal.negate();
+            boolean negative = intVal.signum() == -1;
 
-                out.unsafeWriteInt(val.scale() | 0x80000000);
-            }
-            else
-                out.unsafeWriteInt(val.scale());
+            if (negative)
+                intVal = intVal.negate();
 
             byte[] vals = intVal.toByteArray();
 
+            if (negative)
+                vals[0] |= -0x80;
+
             out.unsafeWriteInt(vals.length);
             out.writeByteArray(vals);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index cd8a487d6d876..31d6f31eef961 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -29,6 +29,7 @@
 import java.lang.reflect.Proxy;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.math.RoundingMode;
 import java.net.InetSocketAddress;
 import java.sql.Timestamp;
 import java.util.AbstractQueue;
@@ -178,6 +179,35 @@ public void testDecimal() throws Exception {
         assertEquals((val = new BigDecimal(new BigInteger("-79228162514264337593543950336"))), marshalUnmarshal(val));
     }
 
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -1)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -2)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -3)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -4)), marshalUnmarshal(val));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleRoundingModeDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.ZERO.setScale(-1, RoundingMode.HALF_UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE).setScale(-3, RoundingMode.HALF_DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE).setScale(-5, RoundingMode.HALF_EVEN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MAX_VALUE).setScale(-8, RoundingMode.UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MIN_VALUE).setScale(-10, RoundingMode.DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MAX_VALUE).setScale(-12, RoundingMode.CEILING)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MIN_VALUE).setScale(-15, RoundingMode.FLOOR)), marshalUnmarshal(val));
+    }
+
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/platforms/cpp/odbc/src/utility.cpp b/modules/platforms/cpp/odbc/src/utility.cpp
index 22191eb8b045e..2c8f9f3a3cdab 100644
--- a/modules/platforms/cpp/odbc/src/utility.cpp
+++ b/modules/platforms/cpp/odbc/src/utility.cpp
@@ -87,8 +87,14 @@ namespace ignite
 
             impl::binary::BinaryUtils::ReadInt8Array(reader.GetStream(), mag.data(), static_cast<int32_t>(mag.size()));
 
-            int32_t sign = (scale & 0x80000000) ? -1 : 1;
-            scale = scale & 0x7FFFFFFF;
+            int32_t sign = 1;
+            
+            if (mag[0] < 0)
+            {
+                mag[0] &= 0x7F;
+
+                sign = -1;
+            }
 
             common::Decimal res(mag.data(), static_cast<int32_t>(mag.size()), scale, sign);
 
@@ -101,14 +107,15 @@ namespace ignite
 
             const common::BigInteger &unscaled = decimal.GetUnscaledValue();
 
-            int32_t signFlag = unscaled.GetSign() == -1 ? 0x80000000 : 0;
-
-            writer.WriteInt32(decimal.GetScale() | signFlag);
+            writer.WriteInt32(decimal.GetScale());
 
             common::FixedSizeArray<int8_t> magnitude;
 
             unscaled.MagnitudeToBytes(magnitude);
 
+            if (unscaled.GetSign() == -1)
+                magnitude[0] |= -0x80;
+
             writer.WriteInt32(magnitude.GetSize());
 
             impl::binary::BinaryUtils::WriteInt8Array(writer.GetStream(), magnitude.GetData(), magnitude.GetSize());
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
index 71a4718130475..3a3100a6b3131 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
@@ -817,16 +817,16 @@ public void TestEchoDecimal()
             Assert.AreEqual(val = decimal.Parse("-11,12"), _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { val, val.ToString() }));
 
             // Test echo with overflow.
-            try
-            {
-                _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { null, decimal.MaxValue.ToString() + 1 });
+            var ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, decimal.MaxValue.ToString() + 1}));
 
-                Assert.Fail();
-            }
-            catch (IgniteException)
-            {
-                // No-op.
-            }
+            Assert.AreEqual("Decimal magnitude overflow (must be less than 96 bits): 104", ex.Message);
+
+            // Negative scale. 1E+1 parses to "1 scale -1" on Java side.
+            ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, "1E+1"}));
+
+            Assert.AreEqual("Decimal value scale overflow (must be between 0 and 28): -1", ex.Message);
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
index cc5d8a109ce86..5ce77d0ee4347 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
@@ -893,7 +893,9 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
             // Write scale and negative flag.
             int scale = (vals[3] & 0x00FF0000) >> 16; 
 
-            stream.WriteInt(((vals[3] & 0x80000000) == 0x80000000) ? (int)((uint)scale | 0x80000000) : scale);
+            stream.WriteInt(scale);
+
+            Boolean neg = vals[3] < 0;
 
             if (idx == -1)
             {
@@ -923,13 +925,15 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             if ((part24 & 0x80) == 0x80)
                             {
                                 stream.WriteInt(len + 1);
+                                
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
 
-                                stream.WriteByte(ByteZero);
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len);
 
-                            stream.WriteByte((byte)part24);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part24 | -0x80) : part24));
                             stream.WriteByte((byte)part16);
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
@@ -940,12 +944,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 1);
-
-                            stream.WriteByte((byte)part16);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part16 | -0x80) : part16));
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
                         }
@@ -955,12 +961,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 1);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 2);
-
-                            stream.WriteByte((byte)part8);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part8 | -0x80) : part8));
                             stream.WriteByte((byte)part0);
                         }
                         else
@@ -969,12 +977,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 2);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 3);
 
-                            stream.WriteByte((byte)part0);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part0 | -0x80) : part0));
                         }
                     }
                     else
@@ -997,18 +1007,16 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
         {
             int scale = stream.ReadInt();
 
-            bool neg;
+            bool neg = false;
+
+            byte[] mag = ReadByteArray(stream);
 
-            if (scale < 0)
+            if ((sbyte)mag[0] < 0)
             {
-                scale = scale & 0x7FFFFFFF;
+                mag[0] &= 0x7F;
 
                 neg = true;
             }
-            else
-                neg = false;
-
-            byte[] mag = ReadByteArray(stream);
 
             if (scale < 0 || scale > 28)
                 throw new BinaryObjectException("Decimal value scale overflow (must be between 0 and 28): " + scale);

From 5efefcbf3bb4573cf85e22869109195158cbe81f Mon Sep 17 00:00:00 2001
From: Vyacheslav Daradur <daradurvs@gmail.com>
Date: Fri, 10 Feb 2017 16:51:37 +0300
Subject: [PATCH 331/516] GG-12647: Backport IGNITE-3196 Add support for
 BigDecimals with negative scale in BinaryMarshaller

---
 .../ignite/internal/binary/BinaryUtils.java   | 12 ++---
 .../internal/binary/BinaryWriterExImpl.java   | 14 +++---
 .../binary/BinaryMarshallerSelfTest.java      | 30 +++++++++++++
 modules/platforms/cpp/odbc/src/utility.cpp    | 17 ++++---
 .../Compute/ComputeApiTest.cs                 | 18 ++++----
 .../Impl/Binary/BinaryUtils.cs                | 44 +++++++++++--------
 6 files changed, 92 insertions(+), 43 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 1153d155c0400..6831ef9bb0d66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -65,8 +65,8 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 
 /**
  * Binary utils.
@@ -1193,13 +1193,15 @@ public static BigDecimal doReadDecimal(BinaryInputStream in) {
         int scale = in.readInt();
         byte[] mag = doReadByteArray(in);
 
-        BigInteger intVal = new BigInteger(mag);
+        boolean negative = mag[0] < 0;
 
-        if (scale < 0) {
-            scale &= 0x7FFFFFFF;
+        if (negative)
+            mag[0] &= 0x7F;
 
+        BigInteger intVal = new BigInteger(mag);
+
+        if (negative)
             intVal = intVal.negate();
-        }
 
         return new BigDecimal(intVal, scale);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index adaacdda40676..3289780e3fd0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -403,18 +403,20 @@ public void doWriteDecimal(@Nullable BigDecimal val) {
 
             out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL);
 
+            out.unsafeWriteInt(val.scale());
+
             BigInteger intVal = val.unscaledValue();
 
-            if (intVal.signum() == -1) {
-                intVal = intVal.negate();
+            boolean negative = intVal.signum() == -1;
 
-                out.unsafeWriteInt(val.scale() | 0x80000000);
-            }
-            else
-                out.unsafeWriteInt(val.scale());
+            if (negative)
+                intVal = intVal.negate();
 
             byte[] vals = intVal.toByteArray();
 
+            if (negative)
+                vals[0] |= -0x80;
+
             out.unsafeWriteInt(vals.length);
             out.writeByteArray(vals);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index cd8a487d6d876..31d6f31eef961 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -29,6 +29,7 @@
 import java.lang.reflect.Proxy;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.math.RoundingMode;
 import java.net.InetSocketAddress;
 import java.sql.Timestamp;
 import java.util.AbstractQueue;
@@ -178,6 +179,35 @@ public void testDecimal() throws Exception {
         assertEquals((val = new BigDecimal(new BigInteger("-79228162514264337593543950336"))), marshalUnmarshal(val));
     }
 
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -1)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -2)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -3)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -4)), marshalUnmarshal(val));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleRoundingModeDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.ZERO.setScale(-1, RoundingMode.HALF_UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE).setScale(-3, RoundingMode.HALF_DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE).setScale(-5, RoundingMode.HALF_EVEN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MAX_VALUE).setScale(-8, RoundingMode.UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MIN_VALUE).setScale(-10, RoundingMode.DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MAX_VALUE).setScale(-12, RoundingMode.CEILING)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MIN_VALUE).setScale(-15, RoundingMode.FLOOR)), marshalUnmarshal(val));
+    }
+
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/platforms/cpp/odbc/src/utility.cpp b/modules/platforms/cpp/odbc/src/utility.cpp
index 63454dcdb188e..3222a38386262 100644
--- a/modules/platforms/cpp/odbc/src/utility.cpp
+++ b/modules/platforms/cpp/odbc/src/utility.cpp
@@ -83,8 +83,14 @@ namespace ignite
 
             impl::binary::BinaryUtils::ReadInt8Array(reader.GetStream(), mag.data(), static_cast<int32_t>(mag.size()));
 
-            int32_t sign = (scale & 0x80000000) ? -1 : 1;
-            scale = scale & 0x7FFFFFFF;
+            int32_t sign = 1;
+            
+            if (mag[0] < 0)
+            {
+                mag[0] &= 0x7F;
+
+                sign = -1;
+            }
 
             common::Decimal res(mag.data(), static_cast<int32_t>(mag.size()), scale, sign);
 
@@ -97,14 +103,15 @@ namespace ignite
 
             const common::BigInteger &unscaled = decimal.GetUnscaledValue();
 
-            int32_t signFlag = unscaled.GetSign() == -1 ? 0x80000000 : 0;
-
-            writer.WriteInt32(decimal.GetScale() | signFlag);
+            writer.WriteInt32(decimal.GetScale());
 
             common::FixedSizeArray<int8_t> magnitude;
 
             unscaled.MagnitudeToBytes(magnitude);
 
+            if (unscaled.GetSign() == -1)
+                magnitude[0] |= -0x80;
+
             writer.WriteInt32(magnitude.GetSize());
 
             impl::binary::BinaryUtils::WriteInt8Array(writer.GetStream(), magnitude.GetData(), magnitude.GetSize());
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
index 71a4718130475..3a3100a6b3131 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
@@ -817,16 +817,16 @@ public void TestEchoDecimal()
             Assert.AreEqual(val = decimal.Parse("-11,12"), _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { val, val.ToString() }));
 
             // Test echo with overflow.
-            try
-            {
-                _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { null, decimal.MaxValue.ToString() + 1 });
+            var ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, decimal.MaxValue.ToString() + 1}));
 
-                Assert.Fail();
-            }
-            catch (IgniteException)
-            {
-                // No-op.
-            }
+            Assert.AreEqual("Decimal magnitude overflow (must be less than 96 bits): 104", ex.Message);
+
+            // Negative scale. 1E+1 parses to "1 scale -1" on Java side.
+            ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, "1E+1"}));
+
+            Assert.AreEqual("Decimal value scale overflow (must be between 0 and 28): -1", ex.Message);
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
index e6a3716b934f2..09c3ad46e08d0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
@@ -893,7 +893,9 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
             // Write scale and negative flag.
             int scale = (vals[3] & 0x00FF0000) >> 16; 
 
-            stream.WriteInt(((vals[3] & 0x80000000) == 0x80000000) ? (int)((uint)scale | 0x80000000) : scale);
+            stream.WriteInt(scale);
+
+            Boolean neg = vals[3] < 0;
 
             if (idx == -1)
             {
@@ -923,13 +925,15 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             if ((part24 & 0x80) == 0x80)
                             {
                                 stream.WriteInt(len + 1);
+                                
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
 
-                                stream.WriteByte(ByteZero);
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len);
 
-                            stream.WriteByte((byte)part24);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part24 | -0x80) : part24));
                             stream.WriteByte((byte)part16);
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
@@ -940,12 +944,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 1);
-
-                            stream.WriteByte((byte)part16);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part16 | -0x80) : part16));
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
                         }
@@ -955,12 +961,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 1);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 2);
-
-                            stream.WriteByte((byte)part8);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part8 | -0x80) : part8));
                             stream.WriteByte((byte)part0);
                         }
                         else
@@ -969,12 +977,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 2);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 3);
 
-                            stream.WriteByte((byte)part0);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part0 | -0x80) : part0));
                         }
                     }
                     else
@@ -997,18 +1007,16 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
         {
             int scale = stream.ReadInt();
 
-            bool neg;
+            bool neg = false;
+
+            byte[] mag = ReadByteArray(stream);
 
-            if (scale < 0)
+            if ((sbyte)mag[0] < 0)
             {
-                scale = scale & 0x7FFFFFFF;
+                mag[0] &= 0x7F;
 
                 neg = true;
             }
-            else
-                neg = false;
-
-            byte[] mag = ReadByteArray(stream);
 
             if (scale < 0 || scale > 28)
                 throw new BinaryObjectException("Decimal value scale overflow (must be between 0 and 28): " + scale);

From e4851162baecfab85263961d3186e070ff296361 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 18 Aug 2017 19:34:01 +0300
Subject: [PATCH 332/516] GG-12620 - Back port
 EVT_CACHE_REBALANCE_PART_DATA_LOST changes from 2.x

---
 .../GridCachePartitionExchangeManager.java    |   2 +-
 .../dht/GridClientPartitionTopology.java      |   6 +
 .../dht/GridDhtPartitionTopology.java         |  11 +
 .../dht/GridDhtPartitionTopologyImpl.java     |  51 +++++
 .../GridDhtPartitionsExchangeFuture.java      |  23 +-
 .../GridLostPartitionRebalanceTest.java       | 198 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite3.java     |   2 +
 7 files changed, 291 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index c62ffd2aec79c..e623a2e26368f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -43,7 +43,6 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
@@ -53,6 +52,7 @@
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index ce8e33ffe5776..3cfc2ada747f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -753,6 +754,11 @@ public long lastUpdateSequence() {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void detectLostPartitions(DiscoveryEvent discoEvt) {
+        assert false : "detectLostPartitions should never be called on client topology";
+    }
+
     /** {@inheritDoc} */
     @Override public void checkEvictions() {
         // No-op.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index bdd84b0329ea1..6275378f38b96 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -22,6 +22,7 @@
 import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
@@ -225,6 +226,16 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
         @Nullable Map<Integer, Long> cntrMap,
         boolean checkEvictions);
 
+    /**
+     * Checks if there is at least one owner for each partition in the cache topology.
+     * If not, marks such a partition as LOST.
+     * <p>
+     * This method should be called on topology coordinator after all partition messages are received.
+     *
+     * @param discoEvt Discovery event for which we detect lost partitions.
+     */
+    public void detectLostPartitions(DiscoveryEvent discoEvt);
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 0cea80de02aac..eb6658bd2a01b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
@@ -1234,6 +1235,56 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void detectLostPartitions(DiscoveryEvent discoEvt) {
+        lock.writeLock().lock();
+
+        try {
+            int parts = cctx.affinity().partitions();
+
+            Collection<Integer> lost = null;
+
+            for (int p = 0; p < parts; p++) {
+                boolean foundOwner = false;
+
+                Set<UUID> nodeIds = part2node.get(p);
+
+                if (nodeIds != null) {
+                    for (UUID nodeId : nodeIds) {
+                        GridDhtPartitionMap2 partMap = node2part.get(nodeId);
+
+                        GridDhtPartitionState state = partMap.get(p);
+
+                        if (state == OWNING) {
+                            foundOwner = true;
+
+                            break;
+                        }
+                    }
+                }
+
+                if (!foundOwner) {
+                    if (lost == null)
+                        lost = new HashSet<>(parts - p, 1.0f);
+
+                    lost.add(p);
+                }
+            }
+
+            if (lost != null) {
+                // Update partition state on all nodes.
+                for (Integer part : lost) {
+                    if (cctx.events().isRecordable(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST))
+                        cctx.events().addPreloadEvent(part, EVT_CACHE_REBALANCE_PART_DATA_LOST,
+                            discoEvt.eventNode(), discoEvt.type(), discoEvt.timestamp());
+                }
+            }
+        }
+        finally {
+            lock.writeLock().unlock();
+        }
+    }
+
     /**
      * @param updateSeq Update sequence.
      * @return {@code True} if state changed.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 3a12198a62340..d7737308075ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -33,7 +33,6 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
@@ -45,6 +44,7 @@
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
@@ -1066,6 +1066,11 @@ private void sendPartitions(ClusterNode oldestNode) {
                 }
             }
 
+            if (discoEvt.type() == EVT_NODE_LEFT ||
+                discoEvt.type() == EVT_NODE_FAILED ||
+                discoEvt.type() == EVT_NODE_JOINED)
+                detectLostPartitions();
+
             Map<Integer, Boolean> m = null;
 
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
@@ -1278,6 +1283,19 @@ private void onAffinityInitialized(IgniteInternalFuture<Map<Integer, Map<Integer
         }
     }
 
+    /**
+     * Detect lost partitions.
+     */
+    private void detectLostPartitions() {
+        if (Thread.currentThread().isInterrupted())
+            return;
+
+        for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
+            if (!cacheCtx.isLocal())
+                cacheCtx.topology().detectLostPartitions(discoEvt);
+        }
+    }
+
     /**
      */
     private void onAllReceived() {
@@ -1291,6 +1309,9 @@ private void onAllReceived() {
                 }
             }
 
+            if (discoEvt.type() == EVT_NODE_LEFT || discoEvt.type() == EVT_NODE_FAILED)
+                detectLostPartitions();
+
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
                 if (!cacheCtx.isLocal())
                     cacheCtx.topology().checkEvictions();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java
new file mode 100644
index 0000000000000..e063b1c138f48
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.CacheRebalancingEvent;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
+
+/**
+ *
+ */
+public class GridLostPartitionRebalanceTest extends GridCommonAbstractTest {
+    /** Cache name. */
+    private static final String CACHE_NAME = "cache";
+
+    /** Latch. */
+    private static CountDownLatch latch = new CountDownLatch(4);
+
+    /** Failed flag. */
+    private static boolean failed;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg.setNodeFilter(NODE_FILTER);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        cfg.setIncludeEventTypes(EVT_CACHE_REBALANCE_PART_DATA_LOST);
+
+        Map<IgnitePredicate<? extends Event>, int[]> listeners = new HashMap<>();
+
+        listeners.put(new Listener(), new int[]{EVT_CACHE_REBALANCE_PART_DATA_LOST});
+
+        cfg.setLocalEventListeners(listeners);
+
+        cfg.setClientMode(gridName.contains("client"));
+
+        final Map<String, Object> attrs = new HashMap<>();
+
+        attrs.put("node.name", gridName);
+
+        cfg.setUserAttributes(attrs);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** Filter. */
+    private static final IgnitePredicate NODE_FILTER = new IgnitePredicate<ClusterNode>() {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        @Override public boolean apply(ClusterNode node) {
+            return !"basenode".equals(node.attribute("node.name"));
+        }
+    };
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartDataLostEvent() throws Exception {
+        List<Ignite> srvrs = new ArrayList<>();
+
+        // Client router. It always up, so client is guaranteed to get
+        // event.
+        srvrs.add(startGrid("basenode"));
+
+        Ignite client = startGrid("client");
+
+        srvrs.add(startGrid("server-1"));
+        srvrs.add(startGrid("server-2"));
+        srvrs.add(startGrid("server-3"));
+
+        awaitPartitionMapExchange();
+
+        IgniteCache<Object, Object> cache = client.cache(CACHE_NAME);
+
+        for (int i = 0; i < 10_000; i++)
+            cache.put(i, i);
+
+        // Stop node with 0 partition.
+        ClusterNode node = client.affinity(CACHE_NAME).mapPartitionToNode(0);
+
+        for (Ignite srv : srvrs) {
+            if (node.equals(srv.cluster().localNode())) {
+                srv.close();
+
+                System.out.println(">> Stopped " + srv.name());
+
+                break;
+            }
+        }
+
+        // Check that all nodes (and clients) got notified.
+        assert latch.await(15, TimeUnit.SECONDS) : latch.getCount();
+
+        // Check that exchange was not finished when event received.
+        assertFalse("Exchange was finished when event received.", failed);
+    }
+
+    /**
+     *
+     */
+    private static class Listener implements IgnitePredicate<CacheRebalancingEvent> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** Ignite. */
+        @SuppressWarnings("unused")
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Got. */
+        private final AtomicBoolean got = new AtomicBoolean(false);
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(CacheRebalancingEvent evt) {
+            int part = evt.partition();
+
+            // AtomicBoolean because new owner will produce two events.
+            if (part == 0 && CACHE_NAME.equals(evt.cacheName()) && got.compareAndSet(false, true)) {
+                System.out.println(">> Received event for 0 partition. [node=" + ignite.name() + ", evt=" + evt
+                    + ", thread=" + Thread.currentThread().getName() + ']');
+
+                latch.countDown();
+
+                if (exchangeCompleted(ignite))
+                    failed = true;
+
+                return false;
+            }
+
+            return true;
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @return {@code True} if exchange finished.
+     */
+    private static boolean exchangeCompleted(Ignite ignite) {
+        GridKernalContext ctx = ((IgniteKernal)ignite).context();
+
+        List<GridDhtPartitionsExchangeFuture> futs = ctx.cache().context().exchange().exchangeFutures();
+
+        for (GridDhtPartitionsExchangeFuture fut : futs) {
+            if (!fut.isDone())
+                return false;
+        }
+
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index 0785714c61b0d..6101aa3f20a0a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -58,6 +58,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridLostPartitionRebalanceTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheDaemonNodeReplicatedSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicGetAndTransformStoreSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedBasicApiTest;
@@ -152,6 +153,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
         suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
         suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
+        suite.addTestSuite(GridLostPartitionRebalanceTest.class);
 
         // Test for byte array value special case.
         suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);

From 74484b2017281595ec964433e0e2f7e76805b01b Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Sat, 5 Aug 2017 15:54:00 +0300
Subject: [PATCH 333/516] IGNITE-5860 make client reconnect on router's suspend

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  50 ++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 250 +++++++++---------
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |   2 +-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   2 +-
 .../tcp/TcpClientDiscoverySpiSelfTest.java    | 229 +++++++++++++++-
 .../IgniteSpiDiscoverySelfTestSuite.java      |   3 +
 6 files changed, 387 insertions(+), 149 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 2f00f95938ca0..24e7457b75366 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -26,6 +26,7 @@
 import java.net.SocketTimeoutException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -467,7 +468,8 @@ else if (state == DISCONNECTED) {
     }
 
     /**
-     * @param recon {@code True} if reconnects.
+     * @param prevAddr If reconnect is in progress, then previous address of the router the client was connected to
+     * and {@code null} otherwise.
      * @param timeout Timeout.
      * @return Opened socket or {@code null} if timeout.
      * @throws InterruptedException If interrupted.
@@ -475,9 +477,9 @@ else if (state == DISCONNECTED) {
      * @see TcpDiscoverySpi#joinTimeout
      */
     @SuppressWarnings("BusyWait")
-    @Nullable private T2<SocketStream, Boolean> joinTopology(boolean recon, long timeout)
+    @Nullable private T2<SocketStream, Boolean> joinTopology(InetSocketAddress prevAddr, long timeout)
         throws IgniteSpiException, InterruptedException {
-        Collection<InetSocketAddress> addrs = null;
+        List<InetSocketAddress> addrs = null;
 
         long startTime = U.currentTimeMillis();
 
@@ -506,17 +508,25 @@ else if (state == DISCONNECTED) {
                 }
             }
 
-            Collection<InetSocketAddress> addrs0 = new ArrayList<>(addrs);
+            // process failed node last
+            if (prevAddr != null) {
+                int idx = addrs.indexOf(prevAddr);
 
-            Iterator<InetSocketAddress> it = addrs.iterator();
+                if (idx != -1)
+                    Collections.swap(addrs, idx, 0);
+            }
+
+            Collection<InetSocketAddress> addrs0 = new ArrayList<>(addrs);
 
             boolean wait = false;
 
-            while (it.hasNext()) {
+            for (int i = addrs.size() - 1; i >= 0; i--) {
                 if (Thread.currentThread().isInterrupted())
                     throw new InterruptedException();
 
-                InetSocketAddress addr = it.next();
+                InetSocketAddress addr = addrs.get(i);
+
+                boolean recon = prevAddr != null;
 
                 T3<SocketStream, Integer, Boolean> sockAndRes;
 
@@ -530,7 +540,7 @@ else if (state == DISCONNECTED) {
                 }
 
                 if (sockAndRes == null) {
-                    it.remove();
+                    addrs.remove(i);
 
                     continue;
                 }
@@ -852,8 +862,8 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
     }
 
     /** {@inheritDoc} */
-    @Override protected IgniteSpiThread workerThread() {
-        return msgWorker;
+    @Override protected Collection<IgniteSpiThread> threads() {
+        return Arrays.asList(sockWriter, msgWorker);
     }
 
     /**
@@ -1339,15 +1349,20 @@ private class Reconnector extends IgniteSpiThread {
         private boolean clientAck;
 
         /** */
-        private boolean join;
+        private final boolean join;
+
+        /** */
+        private final InetSocketAddress prevAddr;
 
         /**
          * @param join {@code True} if reconnects during join.
+         * @param prevAddr Address of the node, that this client was previously connected to.
          */
-        protected Reconnector(boolean join) {
+        protected Reconnector(boolean join, InetSocketAddress prevAddr) {
             super(spi.ignite().name(), "tcp-client-disco-reconnector", log);
 
             this.join = join;
+            this.prevAddr = prevAddr;
         }
 
         /**
@@ -1377,7 +1392,7 @@ public void cancel() {
 
             try {
                 while (true) {
-                    T2<SocketStream, Boolean> joinRes = joinTopology(true, timeout);
+                    T2<SocketStream, Boolean> joinRes = joinTopology(prevAddr, timeout);
 
                     if (joinRes == null) {
                         if (join) {
@@ -1612,6 +1627,10 @@ else if (msg instanceof TcpDiscoveryNodeFailedMessage &&
                     }
                     else if (msg instanceof SocketClosedMessage) {
                         if (((SocketClosedMessage)msg).sock == currSock) {
+                            Socket sock = currSock.sock;
+
+                            InetSocketAddress prevAddr = new InetSocketAddress(sock.getInetAddress(), sock.getPort());
+
                             currSock = null;
 
                             boolean join = joinLatch.getCount() > 0;
@@ -1640,8 +1659,7 @@ else if (msg instanceof SocketClosedMessage) {
 
                                     assert reconnector == null;
 
-                                    final Reconnector reconnector = new Reconnector(join);
-                                    this.reconnector = reconnector;
+                                    reconnector = new Reconnector(join, prevAddr);
                                     reconnector.start();
                                 }
                             }
@@ -1811,7 +1829,7 @@ private void tryJoin() throws InterruptedException {
 
             T2<SocketStream, Boolean> joinRes;
             try {
-                joinRes = joinTopology(false, spi.joinTimeout);
+                joinRes = joinTopology(null, spi.joinTimeout);
             }
             catch (IgniteSpiException e) {
                 joinError(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 22cb618623644..84343e46a9521 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -222,6 +222,9 @@ class ServerImpl extends TcpDiscoveryImpl {
     /** Pending custom messages that should not be sent between NodeAdded and NodeAddFinished messages. */
     private Queue<TcpDiscoveryCustomEventMessage> pendingCustomMsgs = new ArrayDeque<>();
 
+    /** Messages history used for client reconnect. */
+    private final EnsuredMessageHistory msgHist = new EnsuredMessageHistory();
+
     /** If non-shared IP finder is used this flag shows whether IP finder contains local address. */
     private boolean ipFinderHasLocAddr;
 
@@ -1638,8 +1641,23 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
     }
 
     /** {@inheritDoc} */
-    @Override protected IgniteSpiThread workerThread() {
-        return msgWorker;
+    @Override protected Collection<IgniteSpiThread> threads() {
+        Collection<IgniteSpiThread> threads;
+
+        synchronized (mux) {
+            threads = new ArrayList<>(readers.size() + clientMsgWorkers.size() + 4);
+            threads.addAll(readers);
+        }
+
+        threads.addAll(clientMsgWorkers.values());
+        threads.add(tcpSrvr);
+        threads.add(ipFinderCleaner);
+        threads.add(msgWorker);
+        threads.add(statsPrinter);
+
+        threads.removeAll(Collections.<IgniteSpiThread>singleton(null));
+
+        return threads;
     }
 
     /**
@@ -2069,7 +2087,9 @@ else if (msg instanceof TcpDiscoveryNodeLeftMessage)
             else if (msg instanceof TcpDiscoveryNodeFailedMessage)
                 clearClientAddFinished(((TcpDiscoveryNodeFailedMessage)msg).failedNodeId());
 
-            msgs.add(msg);
+            synchronized (msgs) {
+                msgs.add(msg);
+            }
         }
 
         /**
@@ -2131,14 +2151,16 @@ private boolean mapsEqual(Map<Integer, byte[]> m1, Map<Integer, byte[]> m2) {
                 // Client connection failed before it received TcpDiscoveryNodeAddedMessage.
                 List<TcpDiscoveryAbstractMessage> res = null;
 
-                for (TcpDiscoveryAbstractMessage msg : msgs) {
-                    if (msg instanceof TcpDiscoveryNodeAddedMessage) {
-                        if (node.id().equals(((TcpDiscoveryNodeAddedMessage)msg).node().id()))
-                            res = new ArrayList<>(msgs.size());
-                    }
+                synchronized (msgs) {
+                    for (TcpDiscoveryAbstractMessage msg : msgs) {
+                        if (msg instanceof TcpDiscoveryNodeAddedMessage) {
+                            if (node.id().equals(((TcpDiscoveryNodeAddedMessage)msg).node().id()))
+                                res = new ArrayList<>(msgs.size());
+                        }
 
-                    if (res != null)
-                        res.add(prepare(msg, node.id()));
+                        if (res != null)
+                            res.add(prepare(msg, node.id()));
+                    }
                 }
 
                 if (log.isDebugEnabled()) {
@@ -2151,20 +2173,26 @@ private boolean mapsEqual(Map<Integer, byte[]> m1, Map<Integer, byte[]> m2) {
                 return res;
             }
             else {
-                if (msgs.isEmpty())
-                    return Collections.emptyList();
+                Collection<TcpDiscoveryAbstractMessage> cp;
 
-                Collection<TcpDiscoveryAbstractMessage> cp = new ArrayList<>(msgs.size());
+                boolean skip;
 
-                boolean skip = true;
+                synchronized (msgs) {
+                    if (msgs.isEmpty())
+                        return Collections.emptyList();
 
-                for (TcpDiscoveryAbstractMessage msg : msgs) {
-                    if (skip) {
-                        if (msg.id().equals(lastMsgId))
-                            skip = false;
+                    cp = new ArrayList<>(msgs.size());
+
+                    skip = true;
+
+                    for (TcpDiscoveryAbstractMessage msg : msgs) {
+                        if (skip) {
+                            if (msg.id().equals(lastMsgId))
+                                skip = false;
+                        }
+                        else
+                            cp.add(prepare(msg, node.id()));
                     }
-                    else
-                        cp.add(prepare(msg, node.id()));
                 }
 
                 cp = !skip ? cp : null;
@@ -2453,9 +2481,6 @@ private class RingMessageWorker extends MessageWorkerAdapter<TcpDiscoveryAbstrac
         /** Pending messages. */
         private final PendingMessages pendingMsgs = new PendingMessages();
 
-        /** Messages history used for client reconnect. */
-        private final EnsuredMessageHistory msgHist = new EnsuredMessageHistory();
-
         /** Last message that updated topology. */
         private TcpDiscoveryAbstractMessage lastMsg;
 
@@ -2663,9 +2688,6 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage)
             else
                 assert false : "Unknown message type: " + msg.getClass().getSimpleName();
 
-            if (ensured && redirectToClients(msg))
-                msgHist.add(msg);
-
             if (msg.senderNodeId() != null && !msg.senderNodeId().equals(getLocalNodeId())) {
                 // Received a message from remote node.
                 onMessageExchanged();
@@ -2698,6 +2720,9 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage)
          */
         private void sendMessageToClients(TcpDiscoveryAbstractMessage msg) {
             if (redirectToClients(msg)) {
+                if (spi.ensured(msg))
+                    msgHist.add(msg);
+
                 byte[] msgBytes = null;
 
                 for (ClientMessageWorker clientMsgWorker : clientMsgWorkers.values()) {
@@ -3849,9 +3874,6 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                 nodeAddedMsg.client(msg.client());
 
                 processNodeAddedMessage(nodeAddedMsg);
-
-                if (nodeAddedMsg.verified())
-                    msgHist.add(nodeAddedMsg);
             }
             else if (sendMessageToRemotes(msg))
                 sendMessageAcrossRing(msg);
@@ -3940,94 +3962,34 @@ private void trySendMessageDirectly(TcpDiscoveryNode node, TcpDiscoveryAbstractM
         }
 
         /**
-         * Processes client reconnect message.
+         * Forward client reconnect message to coordinator.
          *
          * @param msg Client reconnect message.
          */
         private void processClientReconnectMessage(TcpDiscoveryClientReconnectMessage msg) {
             UUID nodeId = msg.creatorNodeId();
-
             UUID locNodeId = getLocalNodeId();
 
-            boolean isLocNodeRouter = locNodeId.equals(msg.routerNodeId());
-
-            if (!msg.verified()) {
-                TcpDiscoveryNode node = ring.node(nodeId);
-
-                assert node == null || node.isClient();
-
-                if (node != null) {
-                    node.clientRouterNodeId(msg.routerNodeId());
-                    node.aliveCheck(spi.maxMissedClientHbs);
-                }
-
-                if (isLocalNodeCoordinator()) {
-                    msg.verify(locNodeId);
-
-                    if (node != null) {
-                        Collection<TcpDiscoveryAbstractMessage> pending = msgHist.messages(msg.lastMessageId(), node);
-
-                        if (pending != null) {
-                            msg.pendingMessages(pending);
-                            msg.success(true);
-
-                            if (log.isDebugEnabled())
-                                log.debug("Accept client reconnect, restored pending messages " +
-                                    "[locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
-                        }
-                        else {
-                            if (log.isDebugEnabled())
-                                log.debug("Failing reconnecting client node because failed to restore pending " +
-                                    "messages [locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
-
-                            TcpDiscoveryNodeFailedMessage nodeFailedMsg = new TcpDiscoveryNodeFailedMessage(locNodeId,
-                                node.id(), node.internalOrder());
-
-                            processNodeFailedMessage(nodeFailedMsg);
-
-                            if (nodeFailedMsg.verified())
-                                msgHist.add(nodeFailedMsg);
-                        }
-                    }
-                    else if (log.isDebugEnabled())
-                        log.debug("Reconnecting client node is already failed [nodeId=" + nodeId + ']');
+            TcpDiscoveryNode node = ring.node(nodeId);
 
-                    if (isLocNodeRouter) {
-                        ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
+            assert node == null || node.isClient();
 
-                        if (wrk != null)
-                            wrk.addMessage(msg);
-                        else if (log.isDebugEnabled())
-                            log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
-                                locNodeId + ", clientNodeId=" + nodeId + ']');
-                    }
-                    else {
-                        if (sendMessageToRemotes(msg))
-                            sendMessageAcrossRing(msg);
-                    }
-                }
-                else {
-                    if (sendMessageToRemotes(msg))
-                        sendMessageAcrossRing(msg);
-                }
+            if (node != null) {
+                node.clientRouterNodeId(msg.routerNodeId());
+                node.aliveCheck(spi.maxMissedClientHbs);
             }
-            else {
-                if (isLocalNodeCoordinator())
-                    addMessage(new TcpDiscoveryDiscardMessage(locNodeId, msg.id(), false));
 
-                if (isLocNodeRouter) {
-                    ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
+            if (!isLocalNodeCoordinator() && sendMessageToRemotes(msg))
+                sendMessageAcrossRing(msg);
 
-                    if (wrk != null)
-                        wrk.addMessage(msg);
-                    else if (log.isDebugEnabled())
-                        log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
-                            locNodeId + ", clientNodeId=" + nodeId + ']');
-                }
-                else {
-                    if (ring.hasRemoteNodes() && !isLocalNodeCoordinator())
-                        sendMessageAcrossRing(msg);
-                }
+            if (msg.verified() && msg.routerNodeId().equals(getLocalNodeId())) {
+                ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
+
+                if (wrk != null)
+                    wrk.addMessage(msg);
+                else if (log.isDebugEnabled())
+                    log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
+                        locNodeId + ", clientNodeId=" + nodeId + ']');
             }
         }
 
@@ -4071,9 +4033,6 @@ private void processNodeAddedMessage(TcpDiscoveryNodeAddedMessage msg) {
 
                     processNodeAddFinishedMessage(addFinishMsg);
 
-                    if (addFinishMsg.verified())
-                        msgHist.add(addFinishMsg);
-
                     addMessage(new TcpDiscoveryDiscardMessage(locNodeId, msg.id(), false));
 
                     return;
@@ -5134,9 +5093,6 @@ private void processHeartbeatMessage(TcpDiscoveryHeartbeatMessage msg) {
                                             locNodeId, clientNode.id(), clientNode.internalOrder());
 
                                         processNodeFailedMessage(nodeFailedMsg);
-
-                                        if (nodeFailedMsg.verified())
-                                            msgHist.add(nodeFailedMsg);
                                     }
                                 }
                             }
@@ -5334,9 +5290,6 @@ private void processCustomMessage(TcpDiscoveryCustomEventMessage msg) {
                                 ackMsg.topologyVersion(msg.topologyVersion());
 
                                 processCustomMessage(ackMsg);
-
-                                if (ackMsg.verified())
-                                    msgHist.add(ackMsg);
                             }
                             catch (IgniteCheckedException e) {
                                 U.error(log, "Failed to marshal discovery custom message.", e);
@@ -5438,12 +5391,8 @@ private void checkPendingCustomMessages() {
             if (joiningEmpty && isLocalNodeCoordinator()) {
                 TcpDiscoveryCustomEventMessage msg;
 
-                while ((msg = pollPendingCustomeMessage()) != null) {
+                while ((msg = pollPendingCustomeMessage()) != null)
                     processCustomMessage(msg);
-
-                    if (msg.verified())
-                        msgHist.add(msg);
-                }
             }
         }
 
@@ -5997,7 +5946,7 @@ else if (msg instanceof TcpDiscoveryClientReconnectMessage) {
                                     if (clientMsgWrk.getState() == State.NEW)
                                         clientMsgWrk.start();
 
-                                    msgWorker.addMessage(msg);
+                                    processClientReconnectMessage((TcpDiscoveryClientReconnectMessage)msg);
 
                                     continue;
                                 }
@@ -6234,6 +6183,67 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage) {
             }
         }
 
+        /**
+         * Processes client reconnect message.
+         *
+         * @param msg Client reconnect message.
+         */
+        private void processClientReconnectMessage(TcpDiscoveryClientReconnectMessage msg) {
+            UUID nodeId = msg.creatorNodeId();
+            UUID locNodeId = getLocalNodeId();
+
+            boolean isLocNodeRouter = locNodeId.equals(msg.routerNodeId());
+
+            if (isLocNodeRouter) {
+                TcpDiscoveryNode node = ring.node(nodeId);
+                ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
+
+                if (wrk != null && node != null) {
+                    if (!msg.verified()) {
+                        msg.verify(getLocalNodeId());
+
+                        Collection<TcpDiscoveryAbstractMessage> pending = msgHist.messages(msg.lastMessageId(), node);
+
+                        if (pending != null) {
+                            msg.success(true);
+                            msg.pendingMessages(pending);
+
+                            TcpDiscoveryClientReconnectMessage msgCp = new TcpDiscoveryClientReconnectMessage(
+                                msg.creatorNodeId(), msg.routerNodeId(), msg.lastMessageId());
+                            msgCp.client(msg.client());
+
+                            msgWorker.addMessage(msgCp);
+
+                            if (log.isDebugEnabled()) {
+                                log.debug("Accept client reconnect, restored pending messages " +
+                                    "[locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
+                            }
+                        }
+                        else {
+                            if (log.isDebugEnabled())
+                                log.debug("Failing reconnecting client node because failed to restore pending " +
+                                    "messages [locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
+
+                            TcpDiscoveryNodeFailedMessage nodeFailedMsg = new TcpDiscoveryNodeFailedMessage(locNodeId,
+                                node.id(), node.internalOrder());
+
+                            msgWorker.addMessage(nodeFailedMsg);
+                        }
+                    }
+                    else
+                        wrk.addMessage(msg);
+                }
+                else if (log.isDebugEnabled())
+                    log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
+                        locNodeId + ", clientNodeId=" + nodeId + ']');
+
+                if (wrk != null)
+                    wrk.addMessage(msg);
+            }
+            else
+                msgWorker.addMessage(msg);
+        }
+
         /**
          * Processes client heartbeat message.
          *
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index cb85dc1c44b86..1fa7139e7e956 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -292,7 +292,7 @@ protected static String threadStatus(Thread t) {
      *
      * @return Worker thread.
      */
-    protected abstract IgniteSpiThread workerThread();
+    protected abstract Collection<IgniteSpiThread> threads();
 
     /**
      * @throws IgniteSpiException If failed.
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 8c7ef99afcbff..a664ae4f91ab9 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1563,7 +1563,7 @@ protected int readReceipt(Socket sock, long timeout) throws IOException {
      *      empty but never null).
      * @throws org.apache.ignite.spi.IgniteSpiException If an error occurs.
      */
-    protected Collection<InetSocketAddress> resolvedAddresses() throws IgniteSpiException {
+    protected List<InetSocketAddress> resolvedAddresses() throws IgniteSpiException {
         List<InetSocketAddress> res = new ArrayList<>();
 
         Collection<InetSocketAddress> addrs;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 419497753bbc2..819108148cf16 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -62,8 +62,8 @@
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
 import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import org.apache.ignite.spi.IgniteSpiThread;
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryClientReconnectMessage;
@@ -82,13 +82,14 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
+import static org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.DFLT_HEARTBEAT_FREQ;
 
 /**
  * Client-based discovery tests.
  */
 public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     /** */
-    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
     /** */
     protected static final AtomicInteger srvIdx = new AtomicInteger();
@@ -123,6 +124,9 @@ public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     /** */
     private static CountDownLatch clientFailedLatch;
 
+    /** */
+    private static CountDownLatch clientReconnectedLatch;
+
     /** */
     private static CountDownLatch msgLatch;
 
@@ -138,6 +142,9 @@ public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     /** */
     protected long netTimeout = TcpDiscoverySpi.DFLT_NETWORK_TIMEOUT;
 
+    /** */
+    protected Integer reconnectCnt;
+
     /** */
     private boolean longSockTimeouts;
 
@@ -208,6 +215,9 @@ else if (gridName.startsWith("client")) {
         disco.setJoinTimeout(joinTimeout);
         disco.setNetworkTimeout(netTimeout);
 
+        if (reconnectCnt != null)
+            disco.setReconnectCount(reconnectCnt);
+
         disco.setClientReconnectDisabled(reconnectDisabled);
 
         if (disco instanceof TestTcpDiscoverySpi)
@@ -254,6 +264,8 @@ protected TcpDiscoverySpi getDiscoverySpi() {
         clientIpFinder = null;
         joinTimeout = TcpDiscoverySpi.DFLT_JOIN_TIMEOUT;
         netTimeout = TcpDiscoverySpi.DFLT_NETWORK_TIMEOUT;
+        maxMissedClientHbs = TcpDiscoverySpi.DFLT_MAX_MISSED_CLIENT_HEARTBEATS;
+        reconnectCnt = null;
         longSockTimeouts = false;
 
         assert G.allGrids().isEmpty();
@@ -534,6 +546,176 @@ public void testClientReconnectOnRouterFail() throws Exception {
         checkNodes(2, 3);
     }
 
+    /**
+     * Client should reconnect to available server without EVT_CLIENT_NODE_RECONNECTED event.
+     *
+     * @throws Exception If failed.
+     */
+    public void testClientReconnectOnRouterSuspend() throws Exception {
+        reconnectAfterSuspend(false);
+    }
+
+    /**
+     * Client should receive all topology updates after reconnect.
+     *
+     * @throws Exception If failed.
+     */
+    public void testClientReconnectOnRouterSuspendTopologyChange() throws Exception {
+        reconnectAfterSuspend(true);
+    }
+
+    /**
+     * @param changeTop If {@code true} topology is changed after client disconnects
+     * @throws Exception if failed.
+     */
+    private void reconnectAfterSuspend(boolean changeTop) throws Exception {
+        reconnectCnt = 2;
+
+        startServerNodes(2);
+
+        Ignite srv0 = G.ignite("server-0");
+        TcpDiscoveryNode srv0Node = (TcpDiscoveryNode)srv0.cluster().localNode();
+
+        TcpDiscoveryNode srv1Node = (TcpDiscoveryNode)G.ignite("server-1").cluster().localNode();
+
+        clientIpFinder = new TcpDiscoveryVmIpFinder();
+
+        clientIpFinder.setAddresses(
+            Collections.singleton("localhost:" + srv0Node.discoveryPort()));
+
+        startClientNodes(1);
+
+        Ignite client = G.ignite("client-0");
+        TcpDiscoveryNode clientNode = (TcpDiscoveryNode)client.cluster().localNode();
+        TestTcpDiscoverySpi clientSpi = (TestTcpDiscoverySpi)client.configuration().getDiscoverySpi();
+
+        UUID clientNodeId = clientNode.id();
+
+        checkNodes(2, 1);
+
+        clientIpFinder.setAddresses(Collections.singleton("localhost:" + srv1Node.discoveryPort()));
+
+        srvFailedLatch = new CountDownLatch(1);
+
+        attachListeners(2, 1);
+
+        log.info("Pausing router");
+
+        TestTcpDiscoverySpi srvSpi = (TestTcpDiscoverySpi)srv0.configuration().getDiscoverySpi();
+
+        int joinedNodesNum = 3;
+        final CountDownLatch srvJoinedLatch = new CountDownLatch(joinedNodesNum);
+
+        if (changeTop) {
+            client.events().localListen(new IgnitePredicate<Event>() {
+                @Override public boolean apply(Event e) {
+                    srvJoinedLatch.countDown();
+
+                    return true;
+                }
+            }, EVT_NODE_JOINED);
+        }
+
+        srvSpi.pauseAll(true);
+
+        if (changeTop)
+            startServerNodes(joinedNodesNum);
+
+        try {
+            await(srvFailedLatch, 60_000);
+
+            if (changeTop)
+                await(srvJoinedLatch, 5000);
+
+            assertEquals("connected", clientSpi.getSpiState());
+            assertEquals(clientNodeId, clientNode.id());
+            assertEquals(srv1Node.id(), clientNode.clientRouterNodeId());
+        }
+        finally {
+            srvSpi.resumeAll();
+        }
+    }
+
+    /**
+     *
+     */
+    public void testReconnectAfterPause() throws Exception {
+        maxMissedClientHbs = 2;
+
+        startServerNodes(2);
+        startClientNodes(1);
+
+        Ignite client = G.ignite("client-0");
+        TestTcpDiscoverySpi clientSpi = (TestTcpDiscoverySpi)client.configuration().getDiscoverySpi();
+
+        clientReconnectedLatch = new CountDownLatch(1);
+
+        attachListeners(0, 1);
+
+        clientSpi.pauseAll(false);
+
+        try {
+            clientSpi.brakeConnection();
+
+            Thread.sleep(maxMissedClientHbs * DFLT_HEARTBEAT_FREQ * 2);
+        }
+        finally {
+            clientSpi.resumeAll();
+        }
+
+        await(clientReconnectedLatch);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testReconnectAfterMassiveTopologyChange() throws Exception {
+        clientIpFinder = IP_FINDER;
+
+        maxMissedClientHbs = 100;
+        netTimeout = 100000;
+
+        int initSrvsNum = 5;
+        int killNum = 2;
+        int iterations = 3;
+
+        startServerNodes(initSrvsNum);
+        startClientNodes(1);
+
+        Ignite client = G.ignite("client-0");
+        TcpDiscoveryNode clientNode = (TcpDiscoveryNode)client.cluster().localNode();
+        TestTcpDiscoverySpi clientSpi = (TestTcpDiscoverySpi)client.configuration().getDiscoverySpi();
+        final UUID clientNodeId = clientNode.id();
+
+        final CountDownLatch srvJoinedLatch = new CountDownLatch(iterations * killNum);
+
+        client.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event e) {
+                srvJoinedLatch.countDown();
+
+                return true;
+            }
+        }, EVT_NODE_JOINED);
+
+        int minAliveSrvId = 0;
+
+        for (int i = 0; i < iterations; i++) {
+            startServerNodes(killNum);
+
+            for (int j = 0; j < killNum; j++) {
+                failServer(minAliveSrvId);
+
+                minAliveSrvId++;
+            }
+
+            Thread.sleep(500);
+        }
+
+        await(srvJoinedLatch);
+        assertEquals("connected", clientSpi.getSpiState());
+        assertEquals(clientNodeId, clientNode.id());
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -1387,17 +1569,16 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
 
         srvSpi.failNode(client.cluster().localNode().id(), null);
 
-        if (changeTop) {
-            Ignite g = startGrid("server-" + srvIdx.getAndIncrement());
+        assertTrue(disconnectLatch.await(5000, MILLISECONDS));
+        assertTrue(failLatch.await(5000, MILLISECONDS));
 
-            srvNodeIds.add(g.cluster().localNode().id());
+        if (changeTop) {
+            startServerNodes(1);
 
             clientSpi.resumeAll();
         }
 
-        assertTrue(disconnectLatch.await(5000, MILLISECONDS));
         assertTrue(reconnectLatch.await(5000, MILLISECONDS));
-        assertTrue(failLatch.await(5000, MILLISECONDS));
         assertTrue(joinLatch.await(5000, MILLISECONDS));
 
         long topVer = changeTop ? 5L : 4L;
@@ -2003,6 +2184,20 @@ private void attachListeners(int srvCnt, int clientCnt) throws Exception {
                 }, EVT_NODE_FAILED);
             }
         }
+
+        if (clientReconnectedLatch != null) {
+            for (int i = 0; i < clientCnt; i++) {
+                G.ignite("client-" + i).events().localListen(new IgnitePredicate<Event>() {
+                    @Override public boolean apply(Event evt) {
+                        info("Reconnected event fired on client: " + evt);
+
+                        clientReconnectedLatch.countDown();
+
+                        return true;
+                    }
+                }, EVT_CLIENT_NODE_RECONNECTED);
+            }
+        }
     }
 
     /**
@@ -2072,7 +2267,16 @@ else if (srvNodeIds.contains(id))
      * @throws InterruptedException If interrupted.
      */
     protected void await(CountDownLatch latch) throws InterruptedException {
-        assertTrue("Latch count: " + latch.getCount(), latch.await(awaitTime(), MILLISECONDS));
+        await(latch, awaitTime());
+    }
+
+    /**
+     * @param latch Latch.
+     * @param timeout Timeout.
+     * @throws InterruptedException If interrupted.
+     */
+    protected void await(CountDownLatch latch, long timeout) throws InterruptedException {
+        assertTrue("Latch count: " + latch.getCount(), latch.await(timeout, MILLISECONDS));
     }
 
     /**
@@ -2283,8 +2487,10 @@ public void pauseSocketWrite() {
         public void pauseAll(boolean suspend) {
             pauseResumeOperation(true, openSockLock, writeLock);
 
-            if (suspend)
-                impl.workerThread().suspend();
+            if (suspend) {
+                for (Thread t : impl.threads())
+                    t.suspend();
+            }
         }
 
         /**
@@ -2293,7 +2499,8 @@ public void pauseAll(boolean suspend) {
         public void resumeAll() {
             pauseResumeOperation(false, openSockLock, writeLock);
 
-            impl.workerThread().resume();
+            for (IgniteSpiThread t : impl.threads())
+                t.resume();
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 12871492d0979..fa470e911790a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.spi.GridTcpSpiForwardingSelfTest;
 import org.apache.ignite.spi.discovery.AuthenticationRestartTest;
+import org.apache.ignite.spi.discovery.tcp.IgniteClientReconnectMassiveShutdownTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest;
@@ -90,6 +91,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
         suite.addTest(new TestSuite(AuthenticationRestartTest.class));
 
+        suite.addTest(new TestSuite(IgniteClientReconnectMassiveShutdownTest.class));
+
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));

From 5b89e7acf86cfa95a963c587e039efd29638c772 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 21 Aug 2017 17:23:36 +0300
Subject: [PATCH 334/516] IGNITE-5860 - Api compliance fix.

---
 .../java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java    | 2 +-
 .../org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 24e7457b75366..6c12ceeb364bc 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -488,7 +488,7 @@ else if (state == DISCONNECTED) {
                 throw new InterruptedException();
 
             while (addrs == null || addrs.isEmpty()) {
-                addrs = spi.resolvedAddresses();
+                addrs = new ArrayList<>(spi.resolvedAddresses());
 
                 if (!F.isEmpty(addrs)) {
                     if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index a664ae4f91ab9..8c7ef99afcbff 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1563,7 +1563,7 @@ protected int readReceipt(Socket sock, long timeout) throws IOException {
      *      empty but never null).
      * @throws org.apache.ignite.spi.IgniteSpiException If an error occurs.
      */
-    protected List<InetSocketAddress> resolvedAddresses() throws IgniteSpiException {
+    protected Collection<InetSocketAddress> resolvedAddresses() throws IgniteSpiException {
         List<InetSocketAddress> res = new ArrayList<>();
 
         Collection<InetSocketAddress> addrs;

From ded6760b116981a4ad5d58571b4cd140eca72966 Mon Sep 17 00:00:00 2001
From: Alexander Fedotov <alexander.fedotoff@gmail.com>
Date: Tue, 22 Aug 2017 17:07:09 +0300
Subject: [PATCH 335/516] IGNITE-6150 ignite-osgi-karaf-licenses.txt is absent
 in a build

---
 modules/osgi-karaf/pom.xml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/modules/osgi-karaf/pom.xml b/modules/osgi-karaf/pom.xml
index 97a9048974c33..9b99823b8e571 100644
--- a/modules/osgi-karaf/pom.xml
+++ b/modules/osgi-karaf/pom.xml
@@ -42,19 +42,6 @@
             </resource>
         </resources>
         <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-resources-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>filter</id>
-                        <phase>generate-resources</phase>
-                        <goals>
-                            <goal>resources</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>build-helper-maven-plugin</artifactId>
@@ -78,6 +65,19 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-resources-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>filter</id>
+                        <phase>generate-resources</phase>
+                        <goals>
+                            <goal>resources</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 </project>

From 0700f8f473fd1e5507e3782f4c4bb9e74ec17947 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Mon, 21 Aug 2017 17:54:12 +0300
Subject: [PATCH 336/516] GG-12588 Backport IGNITE-5943 Communication. Server
 node may reject client connection during massive clients join.

---
 .../tcp/TcpCommunicationSpi.java              |  61 ++++++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  24 ++-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |  10 +
 .../tcp/IgniteClientConnectTest.java          | 172 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   6 +
 5 files changed, 266 insertions(+), 7 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 7a5f7c1f16f3a..0743ea47c3e67 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -65,6 +65,7 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.GridConcurrentFactory;
 import org.apache.ignite.internal.util.GridSpinReadWriteLock;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -129,6 +130,8 @@
 import org.apache.ignite.spi.IgniteSpiTimeoutObject;
 import org.apache.ignite.spi.communication.CommunicationListener;
 import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentLinkedDeque8;
@@ -137,6 +140,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
+import static org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.RecoveryLastReceivedMessage.NEED_WAIT;
 
 /**
  * <tt>TcpCommunicationSpi</tt> is default communication SPI which uses
@@ -301,6 +305,13 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     /** Connection index meta for session. */
     private static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
 
+    /**
+     * Version when client is ready to wait to connect to server (could be needed when client tries to open connection
+     * before it starts being visible for server)
+     */
+    private static final IgniteProductVersion VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT =
+        IgniteProductVersion.fromString("1.8.10");
+
     /** Message tracker meta for session. */
     private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
 
@@ -435,7 +446,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
              * @param ses Session.
              * @param msg Message.
              */
-            private void onFirstMessage(GridNioSession ses, Message msg) {
+            private void onFirstMessage(final GridNioSession ses, Message msg) {
                 UUID sndId;
 
                 ConnectionKey connKey;
@@ -459,10 +470,35 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                 final ClusterNode rmtNode = getSpiContext().node(sndId);
 
                 if (rmtNode == null) {
-                    U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId +
-                        ", ses=" + ses + ']');
+                    DiscoverySpi discoverySpi = ignite.configuration().getDiscoverySpi();
+
+                    assert discoverySpi instanceof TcpDiscoverySpi;
+
+                    TcpDiscoverySpi tcpDiscoverySpi = (TcpDiscoverySpi) discoverySpi;
+
+                    ClusterNode node0 = tcpDiscoverySpi.getNode0(sndId);
+
+                    boolean unknownNode = true;
 
-                    ses.close();
+                    if (node0 != null) {
+                        assert node0.isClient() : node0;
+
+                        if (node0.version().compareTo(VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT) >= 0)
+                            unknownNode = false;
+                    }
+
+                    if (unknownNode) {
+                        U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId + ", ses=" + ses + ']');
+
+                        ses.close();
+                    }
+                    else {
+                        ses.send(new RecoveryLastReceivedMessage(NEED_WAIT)).listen(new CI1<IgniteInternalFuture<?>>() {
+                            @Override public void apply(IgniteInternalFuture<?> fut) {
+                                ses.close();
+                            }
+                        });
+                    }
 
                     return;
                 }
@@ -2858,6 +2894,8 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
             IgniteSpiOperationTimeoutHelper timeoutHelper = new IgniteSpiOperationTimeoutHelper(this);
 
+            int lastWaitingTimeout = 1;
+
             while (!conn) { // Reconnection on handshake timeout.
                 try {
                     SocketChannel ch = SocketChannel.open();
@@ -2920,6 +2958,18 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
                         if (rcvCnt == -1)
                             return null;
+                        else if (rcvCnt == NEED_WAIT) {
+                            recoveryDesc.release();
+
+                            U.closeQuiet(ch);
+
+                            if (lastWaitingTimeout < 60000)
+                                lastWaitingTimeout *= 2;
+
+                            U.sleep(lastWaitingTimeout);
+
+                            continue;
+                        }
                     }
                     finally {
                         if (recoveryDesc != null && rcvCnt == -1)
@@ -4294,6 +4344,9 @@ public static class RecoveryLastReceivedMessage implements Message {
         /** */
         private static final long serialVersionUID = 0L;
 
+        /** Need wait. */
+        static final long NEED_WAIT = -3;
+
         /** */
         private long rcvCnt;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 1c8988acb2ba1..cfa36f1599590 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1785,9 +1785,27 @@ private static void removeMetrics(TcpDiscoveryHeartbeatMessage msg, UUID nodeId)
     }
 
     /**
-     * Thread that cleans IP finder and keeps it in the correct state, unregistering addresses of the nodes that has
-     * left the topology. <p> This thread should run only on coordinator node and will clean IP finder if and only if
-     * {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()} is {@code true}.
+     * Trying get node in any state (visible or not)
+     * @param nodeId Node id.
+     */
+    ClusterNode getNode0(UUID nodeId) {
+        assert nodeId != null;
+
+        UUID locNodeId0 = getLocalNodeId();
+
+        if (locNodeId0 != null && locNodeId0.equals(nodeId))
+            // Return local node directly.
+            return locNode;
+
+        return ring.node(nodeId);
+    }
+
+    /**
+     * Thread that cleans IP finder and keeps it in the correct state, unregistering
+     * addresses of the nodes that has left the topology.
+     * <p>
+     * This thread should run only on coordinator node and will clean IP finder
+     * if and only if {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()} is {@code true}.
      */
     private class IpFinderCleaner extends IgniteSpiThread {
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 3cc4ee53242b2..b505ae373f210 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -437,6 +437,16 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, T
         return impl.getNode(nodeId);
     }
 
+    /**
+     * @param id Id.
+     */
+    public ClusterNode getNode0(UUID id) {
+        if (impl instanceof ServerImpl)
+            return ((ServerImpl)impl).getNode0(id);
+
+        return getNode(id);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean pingNode(UUID nodeId) {
         return impl.pingNode(nodeId);
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java
new file mode 100644
index 0000000000000..b1c4dbf25434d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+
+/**
+ * We emulate that client receive message about joining to topology earlier than some server nodes in topology.
+ * And make this client connect to such servers.
+ * To emulate this we connect client to second node in topology and pause sending message about joining finishing to
+ * third node.
+ */
+public class IgniteClientConnectTest extends GridCommonAbstractTest {
+
+    /** Custom cache name. */
+    private static final String DEFAULT_CACHE_NAME = "default-cache";
+
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** Latch to stop message sending. */
+    private final CountDownLatch latch = new CountDownLatch(1);
+
+    /** Start client flag. */
+    private final AtomicBoolean clientJustStarted = new AtomicBoolean(false);
+
+    /** Instance name. */
+    private String igniteInstanceName;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        this.igniteInstanceName = igniteInstanceName;
+
+        TestTcpDiscoverySpi disco = new TestTcpDiscoverySpi();
+
+        if (igniteInstanceName.equals("client")) {
+            TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
+
+            ipFinder.registerAddresses(Collections.singleton(new InetSocketAddress(InetAddress.getLoopbackAddress(), 47501)));
+
+            disco.setIpFinder(ipFinder);
+        }
+        else
+            disco.setIpFinder(ipFinder);
+
+        disco.setJoinTimeout(2 * 60_000);
+        disco.setSocketTimeout(1000);
+        disco.setNetworkTimeout(2000);
+
+        cfg.setDiscoverySpi(disco);
+
+        CacheConfiguration cacheConfiguration = new CacheConfiguration()
+                .setName(DEFAULT_CACHE_NAME)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setAffinity(new RendezvousAffinityFunction(false, 8))
+                .setBackups(0);
+
+        cfg.setCacheConfiguration(cacheConfiguration);
+
+        return cfg;
+    }
+
+    /**
+     *
+     * @throws Exception If failed.
+     */
+    public void testClientConnectToBigTopology() throws Exception {
+        Ignite ignite = startGrids(3);
+
+        IgniteCache<Object, Object> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        Set<Integer> keys = new HashSet<>();
+
+        for (int i = 0; i < 80; i++) {
+            cache.put(i, i);
+
+            keys.add(i);
+        }
+
+        TcpDiscoveryImpl discovery = ((TestTcpDiscoverySpi) ignite.configuration().getDiscoverySpi()).discovery();
+
+        assertTrue(discovery instanceof ServerImpl);
+
+        IgniteConfiguration clientCfg = getConfiguration("client");
+
+        clientCfg.setClientMode(true);
+
+        clientJustStarted.set(true);
+
+        Ignite client = startGrid(igniteInstanceName, clientCfg);
+
+        latch.countDown();
+
+        System.err.println("GET ALL");
+        client.cache(DEFAULT_CACHE_NAME).getAll(keys);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     *
+     */
+    class TestTcpDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg, long timeout) throws IOException,
+                IgniteCheckedException {
+            if (msg instanceof TcpDiscoveryNodeAddFinishedMessage) {
+                if (msg.senderNodeId() != null && clientJustStarted.get())
+                    try {
+                        latch.await();
+
+                        Thread.sleep(3000);
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    }
+
+                super.writeToSocket(sock, out, msg, timeout);
+            }
+            else
+                super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /**
+         *
+         */
+        TcpDiscoveryImpl discovery() {
+            return impl;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index e6b39f740a0dd..c2913ba6c6a5b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -19,6 +19,8 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.spi.GridTcpSpiForwardingSelfTest;
+import org.apache.ignite.spi.discovery.tcp.IgniteClientConnectTest;
+import org.apache.ignite.spi.discovery.tcp.IgniteClientReconnectMassiveShutdownTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest;
@@ -88,6 +90,10 @@ public static TestSuite suite() throws Exception {
 
         suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
 
+        //Client connect
+        suite.addTest(new TestSuite(IgniteClientConnectTest.class));
+        suite.addTest(new TestSuite(IgniteClientReconnectMassiveShutdownTest.class));
+
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));

From 890dcd128b3ae0829dd1fb96e3b1ab913f70d541 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Wed, 16 Aug 2017 18:06:36 +0300
Subject: [PATCH 337/516] IGNITE-1094: wip

---
 .../apache/ignite/internal/IgniteKernal.java  |   4 +
 .../ignite/internal/IgniteNodeAttributes.java |   3 +
 .../discovery/GridDiscoveryManager.java       |  39 +-
 .../cache/CacheAffinitySharedManager.java     |  71 +++
 .../DynamicCacheChangeFailureMessage.java     | 129 ++++++
 .../GridCachePartitionExchangeManager.java    |  13 +-
 .../processors/cache/GridCacheProcessor.java  |  42 +-
 .../GridDhtPartitionsExchangeFuture.java      | 189 +++++++-
 .../GridDhtPartitionsSingleMessage.java       |  59 ++-
 .../IgniteDynamicCacheStartFailSelfTest.java  | 421 ++++++++++++++++++
 10 files changed, 938 insertions(+), 32 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 28ac0436eab9d..e58811baa6fac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -201,6 +201,7 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CONSISTENCY_CHECK_SKIPPED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DAEMON;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_EXCHANGE_ROLLBACK_SUPPORTED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_GRID_NAME;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IPS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JIT_NAME;
@@ -1493,6 +1494,9 @@ private void fillNodeAttributes(boolean notifyEnabled) throws IgniteCheckedExcep
         if (cfg.getConnectorConfiguration() != null)
             add(ATTR_REST_PORT_RANGE, cfg.getConnectorConfiguration().getPortRange());
 
+        // Exchange rollback is supported
+        add(ATTR_EXCHANGE_ROLLBACK_SUPPORTED, Boolean.TRUE);
+
         // Stick in SPI versions and classes attributes.
         addSpiAttributes(cfg.getCollisionSpi());
         addSpiAttributes(cfg.getSwapSpaceSpi());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
index 436792459af4a..5002c4c31867c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
@@ -166,6 +166,9 @@ public final class IgniteNodeAttributes {
     /** Ignite security compatibility mode. */
     public static final String ATTR_SECURITY_COMPATIBILITY_MODE = ATTR_PREFIX + ".security.compatibility.enabled";
 
+    /** Internal attribute name constant. */
+    public static final String ATTR_EXCHANGE_ROLLBACK_SUPPORTED = ATTR_PREFIX + ".exchange.rollback.supported";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 71b7217bd6afc..1d67869bb478e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -254,7 +254,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
         new ConcurrentHashMap8<>();
 
     /** Map of dynamic cache filters. */
-    private Map<String, CachePredicate> registeredCaches = new HashMap<>();
+    private Map<String, CachePredicate> registeredCachesPreds = new HashMap<>();
 
     /** */
     private final GridSpinBusyLock busyLock = new GridSpinBusyLock();
@@ -312,19 +312,20 @@ public void setCacheFilter(
         boolean nearEnabled,
         CacheMode cacheMode
     ) {
-        if (!registeredCaches.containsKey(cacheName))
-            registeredCaches.put(cacheName, new CachePredicate(filter, nearEnabled, cacheMode));
+        if (!registeredCachesPreds.containsKey(cacheName))
+            registeredCachesPreds.put(cacheName, new CachePredicate(filter, nearEnabled, cacheMode));
     }
 
     /**
      * Removes dynamic cache filter.
      *
      * @param cacheName Cache name.
+     * @param force Removes dynamic cache filter without check that it was previously created.
      */
-    public void removeCacheFilter(String cacheName) {
-        CachePredicate p = registeredCaches.remove(cacheName);
+    public void removeCacheFilter(String cacheName, boolean force) {
+        CachePredicate p = registeredCachesPreds.remove(cacheName);
 
-        assert p != null : cacheName;
+        assert force || p != null : cacheName;
     }
 
     /**
@@ -336,7 +337,7 @@ public void removeCacheFilter(String cacheName) {
      * @return {@code True} if new node ID was added.
      */
     public boolean addClientNode(String cacheName, UUID clientNodeId, boolean nearEnabled) {
-        CachePredicate p = registeredCaches.get(cacheName);
+        CachePredicate p = registeredCachesPreds.get(cacheName);
 
         assert p != null : cacheName;
 
@@ -351,7 +352,7 @@ public boolean addClientNode(String cacheName, UUID clientNodeId, boolean nearEn
      * @return {@code True} if existing node ID was removed.
      */
     public boolean onClientCacheClose(String cacheName, UUID clientNodeId) {
-        CachePredicate p = registeredCaches.get(cacheName);
+        CachePredicate p = registeredCachesPreds.get(cacheName);
 
         assert p != null : cacheName;
 
@@ -364,12 +365,12 @@ public boolean onClientCacheClose(String cacheName, UUID clientNodeId) {
     public Map<String, Map<UUID, Boolean>> clientNodesMap() {
         Map<String, Map<UUID, Boolean>> res = null;
 
-        for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
+        for (Map.Entry<String, CachePredicate> entry : registeredCachesPreds.entrySet()) {
             CachePredicate pred = entry.getValue();
 
             if (!F.isEmpty(pred.clientNodes)) {
                 if (res == null)
-                    res = U.newHashMap(registeredCaches.size());
+                    res = U.newHashMap(registeredCachesPreds.size());
 
                 res.put(entry.getKey(), new HashMap<>(pred.clientNodes));
             }
@@ -382,7 +383,7 @@ public Map<String, Map<UUID, Boolean>> clientNodesMap() {
      * @param leftNodeId Left node ID.
      */
     private void updateClientNodes(UUID leftNodeId) {
-        for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
+        for (Map.Entry<String, CachePredicate> entry : registeredCachesPreds.entrySet()) {
             CachePredicate pred = entry.getValue();
 
             pred.onNodeLeft(leftNodeId);
@@ -602,7 +603,7 @@ else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
 
                     locJoin = new GridFutureAdapter<>();
 
-                    registeredCaches.clear();
+                    registeredCachesPreds.clear();
 
                     for (AffinityTopologyVersion histVer : discoCacheHist.keySet()) {
                         Object rmvd = discoCacheHist.remove(histVer);
@@ -1718,7 +1719,7 @@ public Collection<ClusterNode> cacheAffinityNodes(int cacheId, AffinityTopologyV
      * @return {@code True} if node is a cache data node.
      */
     public boolean cacheAffinityNode(ClusterNode node, String cacheName) {
-        CachePredicate pred = registeredCaches.get(cacheName);
+        CachePredicate pred = registeredCachesPreds.get(cacheName);
 
         return pred != null && pred.dataNode(node);
     }
@@ -1729,7 +1730,7 @@ public boolean cacheAffinityNode(ClusterNode node, String cacheName) {
      * @return {@code True} if node has near cache enabled.
      */
     public boolean cacheNearNode(ClusterNode node, String cacheName) {
-        CachePredicate pred = registeredCaches.get(cacheName);
+        CachePredicate pred = registeredCachesPreds.get(cacheName);
 
         return pred != null && pred.nearNode(node);
     }
@@ -1740,7 +1741,7 @@ public boolean cacheNearNode(ClusterNode node, String cacheName) {
      * @return {@code True} if node has client cache (without near cache).
      */
     public boolean cacheClientNode(ClusterNode node, String cacheName) {
-        CachePredicate pred = registeredCaches.get(cacheName);
+        CachePredicate pred = registeredCachesPreds.get(cacheName);
 
         return pred != null && pred.clientNode(node);
     }
@@ -1751,7 +1752,7 @@ public boolean cacheClientNode(ClusterNode node, String cacheName) {
      * @return If cache with the given name is accessible on the given node.
      */
     public boolean cacheNode(ClusterNode node, String cacheName) {
-        CachePredicate pred = registeredCaches.get(cacheName);
+        CachePredicate pred = registeredCachesPreds.get(cacheName);
 
         return pred != null && pred.cacheNode(node);
     }
@@ -1761,9 +1762,9 @@ public boolean cacheNode(ClusterNode node, String cacheName) {
      * @return Public cache names accessible on the given node.
      */
     public Map<String, CacheMode> nodeCaches(ClusterNode node) {
-        Map<String, CacheMode> caches = U.newHashMap(registeredCaches.size());
+        Map<String, CacheMode> caches = U.newHashMap(registeredCachesPreds.size());
 
-        for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
+        for (Map.Entry<String, CachePredicate> entry : registeredCachesPreds.entrySet()) {
             String cacheName = entry.getKey();
 
             CachePredicate pred = entry.getValue();
@@ -2012,7 +2013,7 @@ public void reconnect() {
             assert node.order() != 0 : "Invalid node order [locNode=" + loc + ", node=" + node + ']';
             assert !node.isDaemon();
 
-            for (Map.Entry<String, CachePredicate> entry : registeredCaches.entrySet()) {
+            for (Map.Entry<String, CachePredicate> entry : registeredCachesPreds.entrySet()) {
                 String cacheName = entry.getKey();
                 CachePredicate filter = entry.getValue();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 84372febb2c91..fc9e02ce294ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -322,6 +322,77 @@ public void onCacheCreated(GridCacheContext cctx) {
         }
     }
 
+    /**
+     * Called during exchange rollback in order to stop the given cache(s)
+     * even if it's not fully initialized (e.g. fail on cache init stage).
+     *
+     * @param fut Exchange future.
+     * @param crd Coordinator flag.
+     * @param reqs Cache change requests.
+     */
+    public void forceCloseCache(final GridDhtPartitionsExchangeFuture fut, boolean crd,
+        Collection<DynamicCacheChangeRequest> reqs) {
+
+        assert !F.isEmpty(reqs) : fut;
+
+        for (DynamicCacheChangeRequest req : reqs) {
+            assert req.stop() : req;
+
+            Integer cacheId = CU.cacheId(req.cacheName());
+
+            registeredCaches.remove(cacheId);
+        }
+
+        Set<Integer> stoppedCaches = null;
+
+        for (DynamicCacheChangeRequest req : reqs) {
+            Integer cacheId = CU.cacheId(req.cacheName());
+
+            cctx.cache().blockGateway(req);
+
+            if (crd) {
+                CacheHolder cache = caches.remove(cacheId);
+
+                if (cache != null) {
+                    if (stoppedCaches == null)
+                        stoppedCaches = new HashSet<>();
+
+                    stoppedCaches.add(cache.cacheId());
+
+                    cctx.io().removeHandler(cacheId, GridDhtAffinityAssignmentResponse.class);
+                }
+            }
+        }
+
+        if (stoppedCaches != null) {
+            boolean notify = false;
+
+            synchronized (mux) {
+                if (waitInfo != null) {
+                    for (Integer cacheId : stoppedCaches) {
+                        boolean rmv =  waitInfo.waitCaches.remove(cacheId) != null;
+
+                        if (rmv) {
+                            notify = true;
+
+                            waitInfo.assignments.remove(cacheId);
+                        }
+                    }
+                }
+            }
+
+            if (notify) {
+                final AffinityTopologyVersion topVer = affCalcVer;
+
+                cctx.kernalContext().closure().runLocalSafe(new Runnable() {
+                    @Override public void run() {
+                        onCacheStopped(topVer);
+                    }
+                });
+            }
+        }
+    }
+
     /**
      * Called on exchange initiated for cache start/stop request.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
new file mode 100644
index 0000000000000..162c67351cb32
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * This class represents discovery message that is used to provide information about dynamic cache start failure.
+ */
+public class DynamicCacheChangeFailureMessage implements DiscoveryCustomMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Change requests. */
+    @GridToStringInclude
+    private Collection<DynamicCacheChangeRequest> reqs;
+
+    /** Custom message ID. */
+    private IgniteUuid id;
+
+    /** */
+    private GridDhtPartitionExchangeId exchId;
+
+    /** */
+    private GridCacheVersion lastVer;
+
+    @GridToStringInclude
+    private IgniteCheckedException cause;
+
+    /**
+     * Creates new DynamicCacheChangeFailureMessage instance.
+     *
+     * @param exchId Exchange Id.
+     * @param lastVer Last version.
+     * @param cause
+     * @param reqs Cache change requests.
+     */
+    public DynamicCacheChangeFailureMessage(
+        ClusterNode localNode,
+        GridDhtPartitionExchangeId exchId,
+        @Nullable GridCacheVersion lastVer,
+        IgniteCheckedException cause,
+        Collection<DynamicCacheChangeRequest> reqs)
+    {
+        this.id = IgniteUuid.fromUuid(localNode.id());
+        this.exchId = exchId;
+        this.lastVer = lastVer;
+        this.cause = cause;
+        this.reqs = reqs;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteUuid id() {
+        return id;
+    }
+
+    /**
+     * @param id Message ID.
+     */
+    public void id(IgniteUuid id) {
+        this.id = id;
+    }
+
+    /**
+     * @return Collection of change requests.
+     */
+    public Collection<DynamicCacheChangeRequest> requests() {
+        return reqs;
+    }
+
+    public IgniteCheckedException getError() {
+        return cause;
+    }
+
+    /**
+     * @return Last used version among all nodes.
+     */
+    @Nullable public GridCacheVersion lastVersion() {
+        return lastVer;
+    }
+
+    /**
+     * @return Exchange version.
+     */
+    @Nullable public GridDhtPartitionExchangeId exchangeId() {
+        return exchId;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public DiscoveryCustomMessage ackMessage() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMutable() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DynamicCacheChangeFailureMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index c62ffd2aec79c..a994bd46f1f71 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -269,8 +269,17 @@ else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
                                 exchFut = exchangeFuture(exchId, evt, cache, null, msg);
                             }
                         }
-                        else if (msg.exchangeId().topologyVersion().topologyVersion() >= affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
-                            exchangeFuture(msg.exchangeId(), null, null, null, null).onAffinityChangeMessage(customEvt.eventNode(), msg);
+                        else if (msg.exchangeId().topologyVersion().topologyVersion() >=
+                            affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                            exchangeFuture(msg.exchangeId(), null, null, null, null)
+                                .onAffinityChangeMessage(customEvt.eventNode(), msg);
+                    }
+                    else if (customEvt.customMessage() instanceof DynamicCacheChangeFailureMessage) {
+                        DynamicCacheChangeFailureMessage msg =
+                            (DynamicCacheChangeFailureMessage)customEvt.customMessage();
+
+                        exchangeFuture(msg.exchangeId(), null, null, null, null)
+                            .onDynamicCacheChangeFail(customEvt.eventNode(), msg);
                     }
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 35074e9ba7cc7..2fd181dba75b8 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1747,6 +1747,20 @@ public void blockGateway(DynamicCacheChangeRequest req) {
         }
     }
 
+    /**
+     * Called during exchange rollback in order to stop the given cache
+     * even if it's not fully initialized (e.g. fail on cache init stage).
+     */
+     public void forceCloseCache(DynamicCacheChangeRequest req) {
+        assert req.stop() : req;
+
+        registeredCaches.remove(maskNull(req.cacheName()));
+
+        stopGateway(req);
+
+        prepareCacheStop(req, true);
+    }
+
     /**
      * @param req Request.
      */
@@ -1850,7 +1864,17 @@ else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId()) || forc
                         }
                     }
                 }
+            }
+        }
+    }
 
+    /**
+     * @param reqs Collection of requests to complete future for.
+     * @param err Error to be passed to futures.
+     */
+    public void completeStartFutures(Collection<DynamicCacheChangeRequest> reqs, @Nullable Throwable err) {
+        if (!F.isEmpty(reqs)) {
+            for (DynamicCacheChangeRequest req : reqs) {
                 completeStartFuture(req, err);
             }
         }
@@ -2597,10 +2621,26 @@ public boolean onCustomEvent(DiscoveryCustomMessage msg,
         AffinityTopologyVersion topVer) {
         if (msg instanceof CacheAffinityChangeMessage)
             return sharedCtx.affinity().onCustomEvent(((CacheAffinityChangeMessage)msg));
+        else if (msg instanceof DynamicCacheChangeFailureMessage)
+            return onCacheChangeRequested((DynamicCacheChangeFailureMessage)msg);
 
         return msg instanceof DynamicCacheChangeBatch && onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer);
     }
 
+    /**
+     * @param failMsg Dynamic change change request fail message.
+     * @return {@code True} if minor topology version should be increased.
+     */
+    private boolean onCacheChangeRequested(DynamicCacheChangeFailureMessage failMsg) {
+        for (DynamicCacheChangeRequest req : failMsg.requests()) {
+            registeredCaches.remove(maskNull(req.cacheName()));
+
+            ctx.discovery().removeCacheFilter(req.cacheName(), true);
+        }
+
+        return false;
+    }
+
     /**
      * @param batch Change request batch.
      * @param topVer Current topology version.
@@ -2738,7 +2778,7 @@ private boolean onCacheChangeRequested(DynamicCacheChangeBatch batch,
 
                         assert old != null : "Dynamic cache map was concurrently modified [req=" + req + ']';
 
-                        ctx.discovery().removeCacheFilter(req.cacheName());
+                        ctx.discovery().removeCacheFilter(req.cacheName(), false);
 
                         needExchange = true;
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index d3e3701936786..6a7145442f49e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -51,6 +51,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
+import org.apache.ignite.internal.processors.cache.DynamicCacheChangeFailureMessage;
 import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -74,6 +75,7 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteRunnable;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -82,6 +84,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_EXCHANGE_ROLLBACK_SUPPORTED;
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
 
@@ -161,10 +164,10 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
      * Messages received on non-coordinator are stored in case if this node
      * becomes coordinator.
      */
-    private final Map<ClusterNode, GridDhtPartitionsSingleMessage> singleMsgs = new ConcurrentHashMap8<>();
+    private Map<ClusterNode, GridDhtPartitionsSingleMessage> singleMsgs = new ConcurrentHashMap8<>();
 
     /** Messages received from new coordinator. */
-    private final Map<ClusterNode, GridDhtPartitionsFullMessage> fullMsgs = new ConcurrentHashMap8<>();
+    private Map<ClusterNode, GridDhtPartitionsFullMessage> fullMsgs = new ConcurrentHashMap8<>();
 
     /** */
     @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"})
@@ -198,6 +201,12 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** */
     private boolean centralizedAff;
 
+    /** Exception that was thrown during init phase on local node. */
+    private IgniteCheckedException exchangeLocalException;
+
+    /** Exchange exceptions from all participating nodes. */
+    private Map<UUID, Exception> exchangeGlobalExceptions = new ConcurrentHashMap8<>();
+
     /** Forced Rebalance future. */
     private GridCompoundFuture<Boolean, Boolean> forcedRebFut;
 
@@ -536,11 +545,46 @@ public void init() throws IgniteInterruptedCheckedException {
         catch (IgniteNeedReconnectException e) {
             onDone(e);
         }
+        catch (Exception e) {
+            if (reconnectOnError(e)) {
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+
+                return;
+            }
+
+            U.error(log, "Failed to reinitialize local partitions (rebalancing will be stopped): " + exchId, e);
+
+            if (cctx.kernalContext().clientNode() || !isRollbackSupported()) {
+                onDone(e);
+
+                return;
+            }
+
+            exchangeLocalException = new IgniteCheckedException(
+                "Failed to initialize exchange locally [locNodeId=" + cctx.localNodeId() + "]", e);
+
+            exchangeGlobalExceptions.put(cctx.localNodeId(), exchangeLocalException);
+
+            if (crd.isLocal()) {
+                boolean isRemainingEmpty = false;
+
+                synchronized (mux) {
+                    isRemainingEmpty = remaining.isEmpty();
+                }
+
+                if (isRemainingEmpty)
+                    onAllReceived(false);
+            }
+            else
+                sendPartitions(crd);
+
+            initDone();
+        }
         catch (Throwable e) {
             if (reconnectOnError(e))
                 onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
             else {
-                U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
+                U.error(log, "Failed to reinitialize local partitions (rebalancing will be stopped): " + exchId, e);
 
                 onDone(e);
             }
@@ -977,6 +1021,9 @@ private void sendLocalPartitions(ClusterNode node)
             clientOnlyExchange,
             true);
 
+        if (exchangeLocalException != null)
+            m.setError(exchangeLocalException);
+
         if (log.isDebugEnabled())
             log.debug("Sending local partitions [nodeId=" + node.id() + ", exchId=" + exchId + ", msg=" + m + ']');
 
@@ -1084,6 +1131,8 @@ private void sendPartitions(ClusterNode oldestNode) {
 
         cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err, false);
 
+        cctx.cache().completeStartFutures(reqs, err);
+
         if (super.onDone(res, err) && realExchange) {
             if (log.isDebugEnabled())
                 log.debug("Completed partition exchange [localNode=" + cctx.localNodeId() + ", exchange= " + this +
@@ -1130,10 +1179,12 @@ private void sendPartitions(ClusterNode oldestNode) {
      * Cleans up resources to avoid excessive memory usage.
      */
     public void cleanUp() {
-        singleMsgs.clear();
-        fullMsgs.clear();
+        singleMsgs = null;
+        fullMsgs = null;
         crd = null;
         partReleaseFut = null;
+        exchangeLocalException = null;
+        exchangeGlobalExceptions = null;
     }
 
     /**
@@ -1210,6 +1261,9 @@ private void processMessage(ClusterNode node, GridDhtPartitionsSingleMessage msg
 
                     pendingSingleUpdates++;
 
+                    if (msg.getError() != null)
+                        exchangeGlobalExceptions.put(node.id(), msg.getError());
+
                     allReceived = remaining.isEmpty();
                 }
             }
@@ -1278,6 +1332,82 @@ private void onAffinityInitialized(IgniteInternalFuture<Map<Integer, Map<Integer
         }
     }
 
+    /**
+     * Creates an IgniteCheckedException that is used as root cause of the exchange initialization failure.
+     * This method aggregates all the exceptions provided from all participating nodes.
+     *
+     * @param globalExceptions collection exceptions from all participating nodes.
+     * @return exception that represents a cause of the exchange initialization failure.
+     */
+    private IgniteCheckedException createExchangeException(Map<UUID, Exception> globalExceptions) {
+        IgniteCheckedException ex;
+
+        if (exchangeLocalException != null)
+            ex = exchangeLocalException;
+        else
+            ex = new IgniteCheckedException("Failed to complete exchange process (will try to rollback).");
+
+        for (Map.Entry<UUID, Exception> entry : globalExceptions.entrySet()) {
+            // avoid self-suppression
+            if (ex != entry.getValue())
+                ex.addSuppressed(entry.getValue());
+        }
+
+        return ex;
+    }
+
+    /**
+     * Returns {@code true} if the given {@code discoEvt} supports the rollback procedure.
+     *
+     * @return {@code true} if the given {@code discoEvt} supports the rollback procedure.
+     */
+    private boolean isRollbackSupported() {
+        boolean rollbackSupported = false;
+
+        for (ClusterNode node : discoCache.allNodes()) {
+            Boolean exchangeSupported = node.attribute(ATTR_EXCHANGE_ROLLBACK_SUPPORTED);
+            if (exchangeSupported == null || !exchangeSupported)
+                return false;
+        }
+
+        // Currently the rollback process is supported for dynamically started caches.
+        if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && !F.isEmpty(reqs)) {
+            for (DynamicCacheChangeRequest req : reqs) {
+                if (req.start()) {
+                    rollbackSupported = true;
+
+                    break;
+                }
+            }
+        }
+
+        return rollbackSupported;
+    }
+
+    /**
+     * Tries to revert all the changes that were done during initialization phase
+     * in case of the given {@code discoEvt} supports the rollback procedure.
+     */
+    private void rollbackExchange() {
+        if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && !F.isEmpty(reqs)) {
+            for (DynamicCacheChangeRequest req : reqs) {
+                if (req.start()) {
+                    DynamicCacheChangeRequest stopReq =
+                        new DynamicCacheChangeRequest(req.cacheName(), cctx.localNodeId());
+
+                    stopReq.stop(true);
+                    stopReq.deploymentId(req.deploymentId());
+
+                    // cleanup GridCacheProcessor
+                    cctx.cache().forceCloseCache(stopReq);
+
+                    // cleanup CacheAffinitySharedManager
+                    cctx.affinity().forceCloseCache(this, crd.isLocal(), Collections.singletonList(stopReq));
+                }
+            }
+        }
+    }
+
     /**
      * @param discoThread If {@code true} completes future from another thread (to do not block discovery thread).
      */
@@ -1285,6 +1415,24 @@ private void onAllReceived(boolean discoThread) {
         try {
             assert crd.isLocal();
 
+            if (!F.isEmpty(exchangeGlobalExceptions) && isRollbackSupported()) {
+                updateLastVersion(cctx.versions().last());
+
+                cctx.versions().onExchange(lastVer.get().order());
+
+                IgniteCheckedException err = createExchangeException(exchangeGlobalExceptions);
+
+                DynamicCacheChangeFailureMessage msg = new DynamicCacheChangeFailureMessage(
+                    cctx.localNode(), exchId, lastVer.get(), err, reqs);
+
+                if (log.isDebugEnabled())
+                    log.debug("Dynamic cache change failed. Send message to all participating nodes: " + msg);
+
+                cctx.discovery().sendCustomEvent(msg);
+
+                return;
+            }
+
             if (!crd.equals(discoCache.serverNodes().get(0))) {
                 for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
                     if (!cacheCtx.isLocal())
@@ -1488,6 +1636,37 @@ private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
         }
     }
 
+    /**
+     * Cache change failure message callback, processed from the discovery thread.
+     *
+     * @param node Message sender node.
+     * @param msg Message.
+     */
+    public void onDynamicCacheChangeFail(final ClusterNode node, final DynamicCacheChangeFailureMessage msg) {
+        assert exchId.equals(msg.exchangeId()) : msg;
+        assert msg.lastVersion() != null : msg;
+
+        onDiscoveryEvent(new IgniteRunnable() {
+            @Override public void run() {
+                if (isDone() || !enterBusy())
+                    return;
+
+                try {
+                    if (isRollbackSupported())
+                        rollbackExchange();
+
+                    if (!crd.isLocal())
+                        cctx.versions().onExchange(msg.lastVersion().order());
+
+                    onDone(exchId.topologyVersion(), msg.getError());
+                }
+                finally {
+                    leaveBusy();
+                }
+            }
+        });
+    }
+
     /**
      * Affinity change message callback, processed from the same thread as {@link #onNodeLeft}.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index bf08f0abf8832..abc05c0afb83e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -64,6 +64,14 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
     /** Serialized partitions counters. */
     private byte[] partCntrsBytes;
 
+    /** Exception. */
+    @GridToStringInclude
+    @GridDirectTransient
+    private Exception err;
+
+    /** */
+    private byte[] errBytes;
+
     /** */
     private boolean client;
 
@@ -149,6 +157,20 @@ public Map<Integer, Long> partitionUpdateCounters(int cacheId) {
         return Collections.emptyMap();
     }
 
+    /**
+     * @param ex Exception.
+     */
+    public void setError(Exception ex) {
+        this.err = ex;
+    }
+
+    /**
+     * @return Not null exception if exchange processing failed.
+     */
+    @Nullable public Exception getError() {
+        return err;
+    }
+
     /**
      * @return Local partitions.
      */
@@ -156,8 +178,7 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
         return parts;
     }
 
-    /** {@inheritDoc}
-     * @param ctx*/
+    /** {@inheritDoc} */
     @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
         super.prepareMarshal(ctx);
 
@@ -166,6 +187,7 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
         if (marshal) {
             byte[] partsBytes0 = null;
             byte[] partCntrsBytes0 = null;
+            byte[] errBytes0 = null;
 
             if (parts != null && partsBytes == null)
                 partsBytes0 = U.marshal(ctx, parts);
@@ -173,15 +195,20 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
             if (partCntrs != null && partCntrsBytes == null)
                 partCntrsBytes0 = U.marshal(ctx, partCntrs);
 
+            if (err != null && errBytes == null)
+                errBytes0 = U.marshal(ctx, err);
+
             if (compress) {
                 assert !compressed();
 
                 try {
                     byte[] partsBytesZip = U.zip(partsBytes0);
                     byte[] partCntrsBytesZip = U.zip(partCntrsBytes0);
+                    byte[] exBytesZip = U.zip(errBytes0);
 
                     partsBytes0 = partsBytesZip;
                     partCntrsBytes0 = partCntrsBytesZip;
+                    errBytes0 = exBytesZip;
 
                     compressed(true);
                 }
@@ -192,6 +219,7 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
 
             partsBytes = partsBytes0;
             partCntrsBytes = partCntrsBytes0;
+            errBytes = errBytes0;
         }
     }
 
@@ -213,6 +241,13 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
                 partCntrs = U.unmarshal(ctx, partCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
         }
 
+        if (errBytes != null && err == null) {
+            if (compressed())
+                err = U.unmarshalZip(ctx.marshaller(), errBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+            else
+                err = U.unmarshal(ctx, errBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+        }
+
         if (dupPartsData != null) {
             assert parts != null;
 
@@ -262,12 +297,18 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
                 writer.incrementState();
 
             case 8:
-                if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
+                if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
             case 9:
+                if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
+                    return false;
+
+                writer.incrementState();
+
+            case 10:
                 if (!writer.writeByteArray("partsBytes", partsBytes))
                     return false;
 
@@ -306,7 +347,7 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
                 reader.incrementState();
 
             case 8:
-                partCntrsBytes = reader.readByteArray("partCntrsBytes");
+                errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
                     return false;
@@ -314,6 +355,14 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
                 reader.incrementState();
 
             case 9:
+                partCntrsBytes = reader.readByteArray("partCntrsBytes");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 10:
                 partsBytes = reader.readByteArray("partsBytes");
 
                 if (!reader.isLastRead())
@@ -333,7 +382,7 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 10;
+        return 11;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java
new file mode 100644
index 0000000000000..fba54aa80ed92
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import javax.cache.CacheException;
+import javax.cache.configuration.Factory;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.AffinityFunctionContext;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+@SuppressWarnings("unchecked")
+public class IgniteDynamicCacheStartFailSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final String DYNAMIC_CACHE_NAME = "TestDynamicCache";
+
+    /** Coordinator node. */
+    private Ignite crd;
+
+    /** Coordinator node index. */
+    private int crdIdx;
+
+    /**
+     * @return Number of nodes for this test.
+     */
+    public int nodeCount() {
+        return 3;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        crdIdx = 0;
+        crd = startGrid(crdIdx);
+
+        for (int i = 1; i < nodeCount(); ++i)
+            startGrid(i);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    public void testBrokenAffinityFunctionOnAllNodes() {
+        final boolean failOnAllNodes = true;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenAffinityFunctionOnInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 1;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenAffinityFunctionOnNonInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 2;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenAffinityFunctionOnCoordinatorDiffInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = (crdIdx + 1) % nodeCount();
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenAffinityFunctionOnCoordinator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = crdIdx;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenAffinityFunctionWithNodeFilter() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 0;
+        final int numberOfCaches = 1;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFunction(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, true),
+            initiator);
+    }
+
+    public void testBrokenCacheStoreOnAllNodes() {
+        final boolean failOnAllNodes = true;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenCacheStoreOnInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 1;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenCacheStoreOnNonInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = 2;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenCacheStoreOnCoordinatorDiffInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = (crdIdx + 1) % nodeCount();
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testBrokenCacheStoreFunctionOnCoordinator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numberOfCaches = 3;
+        final int initiator = crdIdx;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            initiator);
+    }
+
+    public void testCreateCacheMultipleTimes() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 0;
+        final int numOfAttempts = 100;
+
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFunction(
+            failOnAllNodes, unluckyNode, unluckyCfg, 1, false).get(0);
+
+        for (int i = 0; i < numOfAttempts; ++i) {
+            try {
+                IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+
+                fail("Expected exception was not thrown");
+            }
+            catch (CacheException e) {
+            }
+        }
+    }
+
+    private List<CacheConfiguration> createCacheConfigsWithBrokenAffinityFunction(
+        boolean failOnAllNodes,
+        int unluckyNode,
+        final int unluckyCfg,
+        int cacheNum,
+        boolean useFilter
+        ) {
+        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
+
+        final UUID uuid = ignite(unluckyNode).cluster().localNode().id();
+
+        List<CacheConfiguration> cfgs = new ArrayList<>();
+
+        for (int i = 0; i < cacheNum; ++i) {
+            CacheConfiguration cfg = new CacheConfiguration();
+
+            cfg.setName(DYNAMIC_CACHE_NAME + "-" + i);
+
+            if (i == unluckyCfg)
+                cfg.setAffinity(new BrokenAffinityFunction(failOnAllNodes, getTestGridName(unluckyNode)));
+
+            if (useFilter)
+                cfg.setNodeFilter(new NodeFilter(uuid));
+
+            cfgs.add(cfg);
+        }
+
+        return cfgs;
+    }
+
+    private Collection<CacheConfiguration> createCacheConfigsWithBrokenCacheStore(
+        boolean failOnAllNodes,
+        int unluckyNode,
+        int unluckyCfg,
+        int cacheNum,
+        boolean useFilter
+    ) {
+        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
+
+        final UUID uuid = ignite(unluckyNode).cluster().localNode().id();
+
+        List<CacheConfiguration> cfgs = new ArrayList<>();
+
+        for (int i = 0; i < cacheNum; ++i) {
+            CacheConfiguration cfg = new CacheConfiguration();
+
+            cfg.setName(DYNAMIC_CACHE_NAME + "-" + i);
+
+            if (i == unluckyCfg)
+                cfg.setCacheStoreFactory(new BrokenStoreFactory(failOnAllNodes, getTestGridName(unluckyNode)));
+
+            if (useFilter)
+                cfg.setNodeFilter(new NodeFilter(uuid));
+
+            cfgs.add(cfg);
+        }
+
+        return cfgs;
+    }
+
+    private void testDynamicCacheStart(final Collection<CacheConfiguration> cfgs, final int initiatorId) {
+        assert initiatorId < nodeCount();
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                grid(initiatorId).getOrCreateCaches(cfgs);
+                return null;
+            }
+        }, CacheException.class, null);
+
+        for (CacheConfiguration cfg : cfgs)
+            assertNull("initiatorId=" + initiatorId, grid(initiatorId).cache(cfg.getName()));
+    }
+
+    private static class NodeFilter implements IgnitePredicate<ClusterNode> {
+        /** Cache should be created node with certain UUID. */
+        public UUID uuid;
+
+        public NodeFilter() {
+        }
+
+        public NodeFilter(UUID uuid) {
+            this.uuid = uuid;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode clusterNode) {
+            return clusterNode.id().equals(uuid);
+        }
+    }
+
+    private static class BrokenAffinityFunction extends RendezvousAffinityFunction {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Exception should arise on all nodes. */
+        private boolean exceptionOnAllNodes = false;
+
+        /** Exception should arise on node with certain name. */
+        private String gridName;
+
+        public BrokenAffinityFunction() {
+        }
+
+        public BrokenAffinityFunction(boolean exceptionOnAllNodes, String gridName) {
+            this.exceptionOnAllNodes = exceptionOnAllNodes;
+            this.gridName = gridName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext affCtx) {
+            if (exceptionOnAllNodes || ignite.name().equals(gridName))
+                throw new IllegalStateException("Simulated exception [locNodeId="
+                    + ignite.cluster().localNode().id() + "]");
+            else
+                return super.assignPartitions(affCtx);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(ObjectOutput out) throws IOException {
+            super.writeExternal(out);
+            out.writeBoolean(exceptionOnAllNodes);
+            out.writeObject(gridName);
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("unchecked")
+        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+            super.readExternal(in);
+            exceptionOnAllNodes = in.readBoolean();
+            gridName = (String)in.readObject();
+        }
+    }
+
+    private static class BrokenStoreFactory implements Factory<CacheStore<Integer, String>> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Exception should arise on all nodes. */
+        boolean exceptionOnAllNodes = true;
+
+        /** Exception should arise on node with certain name. */
+        public static String gridName;
+
+        public BrokenStoreFactory() {
+        }
+
+        public BrokenStoreFactory(boolean exceptionOnAllNodes, String gridName) {
+            this.exceptionOnAllNodes = exceptionOnAllNodes;
+            this.gridName = gridName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public CacheStore<Integer, String> create() {
+            if (exceptionOnAllNodes || ignite.name().equals(gridName))
+                throw new IllegalStateException("Simulated exception [locNodeId="
+                    + ignite.cluster().localNode().id() + "]");
+            else
+                return null;
+        }
+    }
+}

From 07e211df9589e037e690c4d0b67034287cfd1317 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 24 Aug 2017 12:12:22 +0300
Subject: [PATCH 338/516] IGNITE-1094: final review

---
 .../managers/discovery/DiscoCache.java        |  19 ++
 .../discovery/GridDiscoveryManager.java       |   5 +-
 .../cache/CacheAffinitySharedManager.java     |  93 ++----
 .../DynamicCacheChangeFailureMessage.java     |  51 ++-
 .../GridCachePartitionExchangeManager.java    |  11 +-
 .../processors/cache/GridCacheProcessor.java  |  77 ++---
 .../GridDhtPartitionsExchangeFuture.java      | 126 +++++---
 .../GridDhtPartitionsSingleMessage.java       |   4 +-
 .../IgniteDynamicCacheStartFailSelfTest.java  | 291 ++++++++++++++----
 .../testsuites/IgniteCacheTestSuite4.java     |   2 +
 10 files changed, 420 insertions(+), 259 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index 5247ac1b28091..30d2b7d9a5c07 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -295,6 +295,25 @@ public void updateAlives(GridDiscoveryManager discovery) {
         }
     }
 
+    /**
+     * Returns {@code True} if all nodes has the given attribute and its value equals to {@code expVal}.
+     *
+     * @param <T> Attribute Type.
+     * @param name Attribute name.
+     * @param expVal Expected value.
+     * @return {@code True} if all the given nodes has the given attribute and its value equals to {@code expVal}.
+     */
+    public <T> boolean checkAttribute(String name, T expVal) {
+        for (ClusterNode node : allNodes) {
+            T attr = node.attribute(name);
+
+            if (attr == null || !expVal.equals(attr))
+                return false;
+        }
+
+        return true;
+    }
+
     /**
      * @param nodes Cluster nodes.
      * @return Empty collection if nodes list is {@code null}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 1d67869bb478e..40dea984ee56d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -320,12 +320,11 @@ public void setCacheFilter(
      * Removes dynamic cache filter.
      *
      * @param cacheName Cache name.
-     * @param force Removes dynamic cache filter without check that it was previously created.
      */
-    public void removeCacheFilter(String cacheName, boolean force) {
+    public void removeCacheFilter(String cacheName) {
         CachePredicate p = registeredCachesPreds.remove(cacheName);
 
-        assert force || p != null : cacheName;
+        assert p != null : cacheName;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index fc9e02ce294ca..7cbd333907058 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -322,6 +322,16 @@ public void onCacheCreated(GridCacheContext cctx) {
         }
     }
 
+    /**
+     * @param cacheId Cache ID.
+     * @return {@code True} if cache is in wait list.
+     */
+    private boolean waitCache(int cacheId) {
+        synchronized (mux) {
+            return waitInfo != null && waitInfo.waitCaches.containsKey(cacheId);
+        }
+    }
+
     /**
      * Called during exchange rollback in order to stop the given cache(s)
      * even if it's not fully initialized (e.g. fail on cache init stage).
@@ -332,7 +342,6 @@ public void onCacheCreated(GridCacheContext cctx) {
      */
     public void forceCloseCache(final GridDhtPartitionsExchangeFuture fut, boolean crd,
         Collection<DynamicCacheChangeRequest> reqs) {
-
         assert !F.isEmpty(reqs) : fut;
 
         for (DynamicCacheChangeRequest req : reqs) {
@@ -341,6 +350,8 @@ public void forceCloseCache(final GridDhtPartitionsExchangeFuture fut, boolean c
             Integer cacheId = CU.cacheId(req.cacheName());
 
             registeredCaches.remove(cacheId);
+
+            assert !waitCache(cacheId);
         }
 
         Set<Integer> stoppedCaches = null;
@@ -363,34 +374,6 @@ public void forceCloseCache(final GridDhtPartitionsExchangeFuture fut, boolean c
                 }
             }
         }
-
-        if (stoppedCaches != null) {
-            boolean notify = false;
-
-            synchronized (mux) {
-                if (waitInfo != null) {
-                    for (Integer cacheId : stoppedCaches) {
-                        boolean rmv =  waitInfo.waitCaches.remove(cacheId) != null;
-
-                        if (rmv) {
-                            notify = true;
-
-                            waitInfo.assignments.remove(cacheId);
-                        }
-                    }
-                }
-            }
-
-            if (notify) {
-                final AffinityTopologyVersion topVer = affCalcVer;
-
-                cctx.kernalContext().closure().runLocalSafe(new Runnable() {
-                    @Override public void run() {
-                        onCacheStopped(topVer);
-                    }
-                });
-            }
-        }
     }
 
     /**
@@ -450,48 +433,38 @@ else if (req.start() && !req.clientStartOnly()) {
             Integer cacheId = CU.cacheId(req.cacheName());
 
             if (req.start()) {
-                try {
-                    cctx.cache().prepareCacheStart(req, fut.topologyVersion());
+                cctx.cache().prepareCacheStart(req, fut.topologyVersion());
 
-                    if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
-                        if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
-                            U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
-                    }
+                if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
+                    if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
+                        U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+                }
 
-                    if (!crd || !lateAffAssign) {
-                        GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+                if (!crd || !lateAffAssign) {
+                    GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
-                        if (cacheCtx != null && !cacheCtx.isLocal()) {
-                            boolean clientCacheStarted =
-                                req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
+                    if (cacheCtx != null && !cacheCtx.isLocal()) {
+                        boolean clientCacheStarted =
+                            req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
 
-                            if (clientCacheStarted)
-                                initAffinity(cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
-                            else if (!req.clientStartOnly()) {
-                                assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
+                        if (clientCacheStarted)
+                            initAffinity(cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
+                        else if (!req.clientStartOnly()) {
+                            assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
 
-                                GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
+                            GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
 
-                                assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
+                            assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
 
-                                List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
-                                    fut.discoveryEvent(), fut.discoCache());
+                            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
+                                fut.discoveryEvent(), fut.discoCache());
 
-                                aff.initialize(fut.topologyVersion(), assignment);
-                            }
+                            aff.initialize(fut.topologyVersion(), assignment);
                         }
                     }
-                    else
-                        initStartedCacheOnCoordinator(fut, cacheId);
-                }
-                catch (IgniteCheckedException | RuntimeException e) {
-                    U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
-                        "[cacheName=" + req.cacheName() + ']', e);
-
-                    cctx.cache().forceCloseCache(fut.topologyVersion(), req, e);
-
-                    throw e;
                 }
+                else
+                    initStartedCacheOnCoordinator(fut, cacheId);
             }
             else if (req.stop() || req.close()) {
                 cctx.cache().blockGateway(req);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
index 162c67351cb32..c755bb1b31f85 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
@@ -18,14 +18,12 @@
 package org.apache.ignite.internal.processors.cache;
 
 import java.util.Collection;
-import java.util.Map;
-import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
@@ -37,9 +35,9 @@ public class DynamicCacheChangeFailureMessage implements DiscoveryCustomMessage
     /** */
     private static final long serialVersionUID = 0L;
 
-    /** Change requests. */
+    /** Cache names. */
     @GridToStringInclude
-    private Collection<DynamicCacheChangeRequest> reqs;
+    private Collection<String> cacheNames;
 
     /** Custom message ID. */
     private IgniteUuid id;
@@ -48,31 +46,31 @@ public class DynamicCacheChangeFailureMessage implements DiscoveryCustomMessage
     private GridDhtPartitionExchangeId exchId;
 
     /** */
-    private GridCacheVersion lastVer;
-
     @GridToStringInclude
     private IgniteCheckedException cause;
 
     /**
      * Creates new DynamicCacheChangeFailureMessage instance.
      *
+     * @param locNode Local node.
      * @param exchId Exchange Id.
-     * @param lastVer Last version.
-     * @param cause
-     * @param reqs Cache change requests.
+     * @param cause Cache start error.
+     * @param cacheNames Cache names.
      */
     public DynamicCacheChangeFailureMessage(
-        ClusterNode localNode,
+        ClusterNode locNode,
         GridDhtPartitionExchangeId exchId,
-        @Nullable GridCacheVersion lastVer,
         IgniteCheckedException cause,
-        Collection<DynamicCacheChangeRequest> reqs)
+        Collection<String> cacheNames)
     {
-        this.id = IgniteUuid.fromUuid(localNode.id());
+        assert exchId != null;
+        assert cause != null;
+        assert !F.isEmpty(cacheNames) : cacheNames;
+
+        this.id = IgniteUuid.fromUuid(locNode.id());
         this.exchId = exchId;
-        this.lastVer = lastVer;
         this.cause = cause;
-        this.reqs = reqs;
+        this.cacheNames = cacheNames;
     }
 
     /** {@inheritDoc} */
@@ -81,30 +79,19 @@ public DynamicCacheChangeFailureMessage(
     }
 
     /**
-     * @param id Message ID.
+     * @return Collection of failed caches.
      */
-    public void id(IgniteUuid id) {
-        this.id = id;
+    public Collection<String> cacheNames() {
+        return cacheNames;
     }
 
     /**
-     * @return Collection of change requests.
+     * @return Cache start error.
      */
-    public Collection<DynamicCacheChangeRequest> requests() {
-        return reqs;
-    }
-
-    public IgniteCheckedException getError() {
+    public IgniteCheckedException error() {
         return cause;
     }
 
-    /**
-     * @return Last used version among all nodes.
-     */
-    @Nullable public GridCacheVersion lastVersion() {
-        return lastVer;
-    }
-
     /**
      * @return Exchange version.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index a994bd46f1f71..4809fd2c0df73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -278,8 +278,10 @@ else if (customEvt.customMessage() instanceof DynamicCacheChangeFailureMessage)
                         DynamicCacheChangeFailureMessage msg =
                             (DynamicCacheChangeFailureMessage)customEvt.customMessage();
 
-                        exchangeFuture(msg.exchangeId(), null, null, null, null)
-                            .onDynamicCacheChangeFail(customEvt.eventNode(), msg);
+                        if (msg.exchangeId().topologyVersion().topologyVersion() >=
+                            affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                            exchangeFuture(msg.exchangeId(), null, null, null, null)
+                                .onDynamicCacheChangeFail(customEvt.eventNode(), msg);
                     }
                 }
 
@@ -1279,6 +1281,9 @@ private void processSinglePartitionUpdate(final ClusterNode node, final GridDhtP
                     log.debug("Received local partition update [nodeId=" + node.id() + ", parts=" +
                         msg + ']');
 
+                if (msg.partitions() == null)
+                    return;
+
                 boolean updated = false;
 
                 for (Map.Entry<Integer, GridDhtPartitionMap2> entry : msg.partitions().entrySet()) {
@@ -1593,7 +1598,7 @@ private void dumpPendingObjects(@Nullable AffinityTopologyVersion exchTopVer) {
      * @return {@code True} if can use compression for partition map messages.
      */
     @SuppressWarnings("SimplifiableIfStatement")
-    private boolean canUsePartitionMapCompression(ClusterNode node) {
+    public boolean canUsePartitionMapCompression(ClusterNode node) {
         IgniteProductVersion ver = node.version();
 
         if (ver.compareToIgnoreTimestamp(GridDhtPartitionsAbstractMessage.PART_MAP_COMPRESS_SINCE) >= 0) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 2fd181dba75b8..673294184a365 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1750,12 +1750,12 @@ public void blockGateway(DynamicCacheChangeRequest req) {
     /**
      * Called during exchange rollback in order to stop the given cache
      * even if it's not fully initialized (e.g. fail on cache init stage).
+     *
+     * @param req Stop request.
      */
      public void forceCloseCache(DynamicCacheChangeRequest req) {
         assert req.stop() : req;
 
-        registeredCaches.remove(maskNull(req.cacheName()));
-
         stopGateway(req);
 
         prepareCacheStop(req, true);
@@ -1795,35 +1795,18 @@ private void prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose)
         }
     }
 
-    /**
-     * Closes cache even if it's not fully initialized (e.g. fail on cache init stage).
-     *
-     * @param topVer Completed topology version.
-     * @param req Change request.
-     * @param err Error.
-     */
-    void forceCloseCache(
-        AffinityTopologyVersion topVer,
-        DynamicCacheChangeRequest req,
-        Throwable err
-    ) {
-        onExchangeDone(topVer, Collections.singleton(req), err, true);
-    }
-
     /**
      * Callback invoked when first exchange future for dynamic cache is completed.
      *
      * @param topVer Completed topology version.
      * @param reqs Change requests.
      * @param err Error.
-     * @param forceClose Close cache despite flags in requests.
      */
     @SuppressWarnings("unchecked")
     public void onExchangeDone(
         AffinityTopologyVersion topVer,
         Collection<DynamicCacheChangeRequest> reqs,
-        Throwable err,
-        boolean forceClose
+        Throwable err
     ) {
         for (GridCacheAdapter<?, ?> cache : caches.values()) {
             GridCacheContext<?, ?> cacheCtx = cache.context();
@@ -1838,43 +1821,35 @@ public void onExchangeDone(
             }
         }
 
-        if (!F.isEmpty(reqs) && (err == null || forceClose)) {
+        if (!F.isEmpty(reqs)) {
             for (DynamicCacheChangeRequest req : reqs) {
-                String masked = maskNull(req.cacheName());
+                if (err == null) {
+                    String masked = maskNull(req.cacheName());
 
-                if (req.stop()) {
-                    stopGateway(req);
+                    if (req.stop()) {
+                        stopGateway(req);
 
-                    prepareCacheStop(req, forceClose);
-                }
-                else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId()) || forceClose) {
-                    IgniteCacheProxy<?, ?> proxy = jCacheProxies.remove(masked);
+                        prepareCacheStop(req, false);
+                    }
+                    else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId())) {
+                        IgniteCacheProxy<?, ?> proxy = jCacheProxies.remove(masked);
 
-                    if (proxy != null) {
-                        if (proxy.context().affinityNode()) {
-                            GridCacheAdapter<?, ?> cache = caches.get(masked);
+                        if (proxy != null) {
+                            if (proxy.context().affinityNode()) {
+                                GridCacheAdapter<?, ?> cache = caches.get(masked);
 
-                            if (cache != null)
-                                jCacheProxies.put(masked, new IgniteCacheProxy(cache.context(), cache, null, false));
-                        }
-                        else {
-                            proxy.context().gate().onStopped();
+                                if (cache != null)
+                                    jCacheProxies.put(masked, new IgniteCacheProxy(cache.context(), cache, null, false));
+                            }
+                            else {
+                                proxy.context().gate().onStopped();
 
-                            prepareCacheStop(req, forceClose);
+                                prepareCacheStop(req, false);
+                            }
                         }
                     }
                 }
-            }
-        }
-    }
 
-    /**
-     * @param reqs Collection of requests to complete future for.
-     * @param err Error to be passed to futures.
-     */
-    public void completeStartFutures(Collection<DynamicCacheChangeRequest> reqs, @Nullable Throwable err) {
-        if (!F.isEmpty(reqs)) {
-            for (DynamicCacheChangeRequest req : reqs) {
                 completeStartFuture(req, err);
             }
         }
@@ -2632,10 +2607,10 @@ else if (msg instanceof DynamicCacheChangeFailureMessage)
      * @return {@code True} if minor topology version should be increased.
      */
     private boolean onCacheChangeRequested(DynamicCacheChangeFailureMessage failMsg) {
-        for (DynamicCacheChangeRequest req : failMsg.requests()) {
-            registeredCaches.remove(maskNull(req.cacheName()));
+        for (String cacheName : failMsg.cacheNames()) {
+            registeredCaches.remove(maskNull(cacheName));
 
-            ctx.discovery().removeCacheFilter(req.cacheName(), true);
+            ctx.discovery().removeCacheFilter(cacheName);
         }
 
         return false;
@@ -2778,7 +2753,7 @@ private boolean onCacheChangeRequested(DynamicCacheChangeBatch batch,
 
                         assert old != null : "Dynamic cache map was concurrently modified [req=" + req + ']';
 
-                        ctx.discovery().removeCacheFilter(req.cacheName(), false);
+                        ctx.discovery().removeCacheFilter(req.cacheName());
 
                         needExchange = true;
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 6a7145442f49e..b4d02ec2bdebd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -75,7 +75,6 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
-import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteRunnable;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -164,10 +163,10 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
      * Messages received on non-coordinator are stored in case if this node
      * becomes coordinator.
      */
-    private Map<ClusterNode, GridDhtPartitionsSingleMessage> singleMsgs = new ConcurrentHashMap8<>();
+    private final Map<ClusterNode, GridDhtPartitionsSingleMessage> singleMsgs = new ConcurrentHashMap8<>();
 
     /** Messages received from new coordinator. */
-    private Map<ClusterNode, GridDhtPartitionsFullMessage> fullMsgs = new ConcurrentHashMap8<>();
+    private final Map<ClusterNode, GridDhtPartitionsFullMessage> fullMsgs = new ConcurrentHashMap8<>();
 
     /** */
     @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"})
@@ -202,10 +201,10 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     private boolean centralizedAff;
 
     /** Exception that was thrown during init phase on local node. */
-    private IgniteCheckedException exchangeLocalException;
+    private IgniteCheckedException exchangeLocE;
 
     /** Exchange exceptions from all participating nodes. */
-    private Map<UUID, Exception> exchangeGlobalExceptions = new ConcurrentHashMap8<>();
+    private final Map<UUID, Exception> exchangeGlobalExceptions = new ConcurrentHashMap8<>();
 
     /** Forced Rebalance future. */
     private GridCompoundFuture<Boolean, Boolean> forcedRebFut;
@@ -552,31 +551,39 @@ public void init() throws IgniteInterruptedCheckedException {
                 return;
             }
 
-            U.error(log, "Failed to reinitialize local partitions (rebalancing will be stopped): " + exchId, e);
+            U.error(log, "Failed to initialize cache (will try to rollback). " + exchId, e);
 
-            if (cctx.kernalContext().clientNode() || !isRollbackSupported()) {
+            if (!isRollbackSupported()) {
                 onDone(e);
 
                 return;
             }
 
-            exchangeLocalException = new IgniteCheckedException(
+            exchangeLocE = new IgniteCheckedException(
                 "Failed to initialize exchange locally [locNodeId=" + cctx.localNodeId() + "]", e);
 
-            exchangeGlobalExceptions.put(cctx.localNodeId(), exchangeLocalException);
+            exchangeGlobalExceptions.put(cctx.localNodeId(), exchangeLocE);
 
-            if (crd.isLocal()) {
-                boolean isRemainingEmpty = false;
+            if (crd != null) {
+                if (crd.isLocal()) {
+                    boolean allRcvd;
 
-                synchronized (mux) {
-                    isRemainingEmpty = remaining.isEmpty();
+                    synchronized (mux) {
+                        allRcvd = remaining.isEmpty();
+                    }
+
+                    if (allRcvd)
+                        onAllReceived(false);
                 }
+                else {
+                    clientOnlyExchange = cctx.kernalContext().clientNode();
 
-                if (isRemainingEmpty)
-                    onAllReceived(false);
+                    if (!centralizedAff)
+                        sendPartitions(crd);
+                }
             }
             else
-                sendPartitions(crd);
+                onDone(e);
 
             initDone();
         }
@@ -1016,13 +1023,21 @@ public boolean stopping(int cacheId) {
      */
     private void sendLocalPartitions(ClusterNode node)
         throws IgniteCheckedException {
-        GridDhtPartitionsSingleMessage m = cctx.exchange().createPartitionsSingleMessage(node,
-            exchangeId(),
-            clientOnlyExchange,
-            true);
+        GridDhtPartitionsSingleMessage m;
 
-        if (exchangeLocalException != null)
-            m.setError(exchangeLocalException);
+        if (exchangeLocE == null)
+            m = cctx.exchange().createPartitionsSingleMessage(node,
+                exchangeId(),
+                clientOnlyExchange,
+                true);
+        else {
+            m = new GridDhtPartitionsSingleMessage(exchangeId(),
+                clientOnlyExchange,
+                cctx.versions().last(),
+                cctx.exchange().canUsePartitionMapCompression(node));
+
+            m.setError(exchangeLocE);
+        }
 
         if (log.isDebugEnabled())
             log.debug("Sending local partitions [nodeId=" + node.id() + ", exchId=" + exchId + ", msg=" + m + ']');
@@ -1129,9 +1144,7 @@ private void sendPartitions(ClusterNode oldestNode) {
 
         cctx.exchange().onExchangeDone(this, err);
 
-        cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err, false);
-
-        cctx.cache().completeStartFutures(reqs, err);
+        cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err);
 
         if (super.onDone(res, err) && realExchange) {
             if (log.isDebugEnabled())
@@ -1179,12 +1192,12 @@ private void sendPartitions(ClusterNode oldestNode) {
      * Cleans up resources to avoid excessive memory usage.
      */
     public void cleanUp() {
-        singleMsgs = null;
-        fullMsgs = null;
+        singleMsgs.clear();
+        fullMsgs.clear();
         crd = null;
         partReleaseFut = null;
-        exchangeLocalException = null;
-        exchangeGlobalExceptions = null;
+        exchangeLocE = null;
+        exchangeGlobalExceptions.clear();
     }
 
     /**
@@ -1222,10 +1235,17 @@ public void onReceive(final ClusterNode node, final GridDhtPartitionsSingleMessa
                 log.debug("Received message for finished future (will reply only to sender) [msg=" + msg +
                     ", fut=" + this + ']');
 
+            // Custom message (DynamicCacheChangeFailureMessage) was sent.
+            // Do not need sendAllPartitions.
+            if (!exchangeGlobalExceptions.isEmpty())
+                return;
+
             if (!centralizedAff)
                 sendAllPartitions(node.id(), cctx.gridConfig().getNetworkSendRetryCount());
         }
         else {
+            assert !msg.client();
+
             initFut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
                 @Override public void apply(IgniteInternalFuture<Boolean> f) {
                     try {
@@ -1342,13 +1362,13 @@ private void onAffinityInitialized(IgniteInternalFuture<Map<Integer, Map<Integer
     private IgniteCheckedException createExchangeException(Map<UUID, Exception> globalExceptions) {
         IgniteCheckedException ex;
 
-        if (exchangeLocalException != null)
-            ex = exchangeLocalException;
+        if (exchangeLocE != null)
+            ex = exchangeLocE;
         else
             ex = new IgniteCheckedException("Failed to complete exchange process (will try to rollback).");
 
         for (Map.Entry<UUID, Exception> entry : globalExceptions.entrySet()) {
-            // avoid self-suppression
+            // Avoid self-suppression.
             if (ex != entry.getValue())
                 ex.addSuppressed(entry.getValue());
         }
@@ -1364,11 +1384,8 @@ private IgniteCheckedException createExchangeException(Map<UUID, Exception> glob
     private boolean isRollbackSupported() {
         boolean rollbackSupported = false;
 
-        for (ClusterNode node : discoCache.allNodes()) {
-            Boolean exchangeSupported = node.attribute(ATTR_EXCHANGE_ROLLBACK_SUPPORTED);
-            if (exchangeSupported == null || !exchangeSupported)
-                return false;
-        }
+        if (!discoCache.checkAttribute(ATTR_EXCHANGE_ROLLBACK_SUPPORTED, Boolean.TRUE))
+            return false;
 
         // Currently the rollback process is supported for dynamically started caches.
         if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && !F.isEmpty(reqs)) {
@@ -1398,10 +1415,10 @@ private void rollbackExchange() {
                     stopReq.stop(true);
                     stopReq.deploymentId(req.deploymentId());
 
-                    // cleanup GridCacheProcessor
+                    // Cleanup GridCacheProcessor.
                     cctx.cache().forceCloseCache(stopReq);
 
-                    // cleanup CacheAffinitySharedManager
+                    // Cleanup CacheAffinitySharedManager.
                     cctx.affinity().forceCloseCache(this, crd.isLocal(), Collections.singletonList(stopReq));
                 }
             }
@@ -1416,14 +1433,15 @@ private void onAllReceived(boolean discoThread) {
             assert crd.isLocal();
 
             if (!F.isEmpty(exchangeGlobalExceptions) && isRollbackSupported()) {
-                updateLastVersion(cctx.versions().last());
+                IgniteCheckedException err = createExchangeException(exchangeGlobalExceptions);
 
-                cctx.versions().onExchange(lastVer.get().order());
+                List<String> cacheNames = new ArrayList<>(reqs.size());
 
-                IgniteCheckedException err = createExchangeException(exchangeGlobalExceptions);
+                for (DynamicCacheChangeRequest req : reqs)
+                    cacheNames.add(req.cacheName());
 
                 DynamicCacheChangeFailureMessage msg = new DynamicCacheChangeFailureMessage(
-                    cctx.localNode(), exchId, lastVer.get(), err, reqs);
+                    cctx.localNode(), exchId, err, cacheNames);
 
                 if (log.isDebugEnabled())
                     log.debug("Dynamic cache change failed. Send message to all participating nodes: " + msg);
@@ -1572,6 +1590,8 @@ private void processMessage(ClusterNode node, GridDhtPartitionsFullMessage msg)
         assert msg.exchangeId().equals(exchId) : msg;
         assert msg.lastVersion() != null : msg;
 
+        boolean isRollbackNedeed = false;
+
         synchronized (mux) {
             if (crd == null)
                 return;
@@ -1586,6 +1606,17 @@ private void processMessage(ClusterNode node, GridDhtPartitionsFullMessage msg)
 
                 return;
             }
+
+            if (exchangeLocE != null && isRollbackSupported())
+                isRollbackNedeed = true;
+        }
+
+        if (isRollbackNedeed) {
+            rollbackExchange();
+
+            onDone(exchId.topologyVersion(), exchangeLocE);
+
+            return;
         }
 
         updatePartitionFullMap(msg);
@@ -1625,6 +1656,9 @@ private void updatePartitionFullMap(GridDhtPartitionsFullMessage msg) {
      * @param msg Partitions single message.
      */
     private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
+        if (msg.partitions() == null)
+            return;
+
         for (Map.Entry<Integer, GridDhtPartitionMap2> entry : msg.partitions().entrySet()) {
             Integer cacheId = entry.getKey();
             GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
@@ -1644,7 +1678,6 @@ private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
      */
     public void onDynamicCacheChangeFail(final ClusterNode node, final DynamicCacheChangeFailureMessage msg) {
         assert exchId.equals(msg.exchangeId()) : msg;
-        assert msg.lastVersion() != null : msg;
 
         onDiscoveryEvent(new IgniteRunnable() {
             @Override public void run() {
@@ -1655,10 +1688,7 @@ public void onDynamicCacheChangeFail(final ClusterNode node, final DynamicCacheC
                     if (isRollbackSupported())
                         rollbackExchange();
 
-                    if (!crd.isLocal())
-                        cctx.versions().onExchange(msg.lastVersion().order());
-
-                    onDone(exchId.topologyVersion(), msg.getError());
+                    onDone(exchId.topologyVersion(), msg.error());
                 }
                 finally {
                     leaveBusy();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index abc05c0afb83e..f2664d50b8f8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -182,7 +182,9 @@ public Map<Integer, GridDhtPartitionMap2> partitions() {
     @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
         super.prepareMarshal(ctx);
 
-        boolean marshal = (parts != null && partsBytes == null) || (partCntrs != null && partCntrsBytes == null);
+        boolean marshal = (parts != null && partsBytes == null) ||
+            (partCntrs != null && partCntrsBytes == null) ||
+            (err != null && errBytes == null);
 
         if (marshal) {
             byte[] partsBytes0 = null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java
index fba54aa80ed92..2ca40249c29d3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailSelfTest.java
@@ -35,8 +35,6 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -45,6 +43,9 @@
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+/**
+ *
+ */
 @SuppressWarnings("unchecked")
 public class IgniteDynamicCacheStartFailSelfTest extends GridCommonAbstractTest {
     /** */
@@ -53,8 +54,8 @@ public class IgniteDynamicCacheStartFailSelfTest extends GridCommonAbstractTest
     /** */
     private static final String DYNAMIC_CACHE_NAME = "TestDynamicCache";
 
-    /** Coordinator node. */
-    private Ignite crd;
+    /** */
+    private static final String CLIENT_GRID_NAME = "client";
 
     /** Coordinator node index. */
     private int crdIdx;
@@ -78,7 +79,8 @@ public int nodeCount() {
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         crdIdx = 0;
-        crd = startGrid(crdIdx);
+
+        startGrid(crdIdx);
 
         for (int i = 1; i < nodeCount(); ++i)
             startGrid(i);
@@ -89,156 +91,307 @@ public int nodeCount() {
         stopAllGrids();
     }
 
-    public void testBrokenAffinityFunctionOnAllNodes() {
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnServerFailedOnClient() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnClient";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-server-1");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, clientName));
+
+        try {
+            IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+        }
+        catch (CacheException e) {
+            fail("Exception should not be thrown.");
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnServerFailedOnServer() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnServer";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-server-2");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, getTestGridName(0)));
+
+        try {
+            IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+
+            fail("Expected exception was not thrown.");
+        }
+        catch (CacheException e) {
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnClientFailOnClient() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnClientFailOnClient";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-client-1");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, clientName));
+
+        try {
+            IgniteCache cache = client.getOrCreateCache(cfg);
+
+            fail("Expected exception was not thrown.");
+        }
+        catch (CacheException e) {
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnClientFailOnServer() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnClientFailOnServer";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-client-2");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, getTestGridName(0)));
+
+        try {
+            IgniteCache cache = client.getOrCreateCache(cfg);
+
+            fail("Expected exception was not thrown.");
+        }
+        catch (CacheException e) {
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on all nodes.
+     */
+    public void testBrokenAffinityFunOnAllNodes() {
         final boolean failOnAllNodes = true;
         final int unluckyNode = 0;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 0;
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenAffinityFunctionOnInitiator() {
+    /**
+     * Test cache start with broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunOnInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 1;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 1;
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenAffinityFunctionOnNonInitiator() {
+    /**
+     * Test cache start with broken affinity function that throws an exception on non-initiator node.
+     */
+    public void testBrokenAffinityFunOnNonInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 1;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 2;
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenAffinityFunctionOnCoordinatorDiffInitiator() {
+    /**
+     * Test cache start with broken affinity function that throws an exception on coordinator node.
+     */
+    public void testBrokenAffinityFunOnCoordinatorDiffInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = crdIdx;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = (crdIdx + 1) % nodeCount();
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenAffinityFunctionOnCoordinator() {
+    /**
+     * Test cache start with broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunOnCoordinator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = crdIdx;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = crdIdx;
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenAffinityFunctionWithNodeFilter() {
+    /**
+     * Tests cache start with node filter and broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunWithNodeFilter() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 0;
         final int unluckyCfg = 0;
-        final int numberOfCaches = 1;
+        final int numOfCaches = 1;
         final int initiator = 0;
 
         testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFunction(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, true),
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, true),
             initiator);
     }
 
+    /**
+     * Tests cache start with broken cache store that throws an exception on all nodes.
+     */
     public void testBrokenCacheStoreOnAllNodes() {
         final boolean failOnAllNodes = true;
         final int unluckyNode = 0;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 0;
 
         testDynamicCacheStart(
             createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
+    /**
+     * Tests cache start with broken cache store that throws an exception on initiator node.
+     */
     public void testBrokenCacheStoreOnInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 1;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 1;
 
         testDynamicCacheStart(
             createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
+    /**
+     * Tests cache start with broken cache store that throws an exception on non-initiator node.
+     */
     public void testBrokenCacheStoreOnNonInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 1;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = 2;
 
         testDynamicCacheStart(
             createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
+    /**
+     *  Tests cache start with broken cache store that throws an exception on initiator node.
+     */
     public void testBrokenCacheStoreOnCoordinatorDiffInitiator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = crdIdx;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = (crdIdx + 1) % nodeCount();
 
         testDynamicCacheStart(
             createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
-    public void testBrokenCacheStoreFunctionOnCoordinator() {
+    /**
+     *  Tests cache start with broken cache store that throws an exception on coordinator node.
+     */
+    public void testBrokenCacheStoreFunOnCoordinator() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = crdIdx;
         final int unluckyCfg = 1;
-        final int numberOfCaches = 3;
+        final int numOfCaches = 3;
         final int initiator = crdIdx;
 
         testDynamicCacheStart(
             createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numberOfCaches, false),
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
             initiator);
     }
 
+    /**
+     *  Tests multiple creation of cache with broken affinity function.
+     */
     public void testCreateCacheMultipleTimes() {
         final boolean failOnAllNodes = false;
         final int unluckyNode = 1;
         final int unluckyCfg = 0;
         final int numOfAttempts = 100;
 
-        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFunction(
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
             failOnAllNodes, unluckyNode, unluckyCfg, 1, false).get(0);
 
         for (int i = 0; i < numOfAttempts; ++i) {
@@ -252,7 +405,7 @@ public void testCreateCacheMultipleTimes() {
         }
     }
 
-    private List<CacheConfiguration> createCacheConfigsWithBrokenAffinityFunction(
+    private List<CacheConfiguration> createCacheConfigsWithBrokenAffinityFun(
         boolean failOnAllNodes,
         int unluckyNode,
         final int unluckyCfg,
@@ -321,18 +474,18 @@ private void testDynamicCacheStart(final Collection<CacheConfiguration> cfgs, fi
                 return null;
             }
         }, CacheException.class, null);
-
-        for (CacheConfiguration cfg : cfgs)
-            assertNull("initiatorId=" + initiatorId, grid(initiatorId).cache(cfg.getName()));
     }
 
+    /**
+     * Filter specifying on which node the cache should be started.
+     */
     private static class NodeFilter implements IgnitePredicate<ClusterNode> {
         /** Cache should be created node with certain UUID. */
         public UUID uuid;
 
-        public NodeFilter() {
-        }
-
+        /**
+         * @param uuid node ID.
+         */
         public NodeFilter(UUID uuid) {
             this.uuid = uuid;
         }
@@ -343,6 +496,9 @@ public NodeFilter(UUID uuid) {
         }
     }
 
+    /**
+     * Factory that throws an exception is got created.
+     */
     private static class BrokenAffinityFunction extends RendezvousAffinityFunction {
         /** */
         private static final long serialVersionUID = 0L;
@@ -352,22 +508,30 @@ private static class BrokenAffinityFunction extends RendezvousAffinityFunction {
         private Ignite ignite;
 
         /** Exception should arise on all nodes. */
-        private boolean exceptionOnAllNodes = false;
+        private boolean eOnAllNodes = false;
 
         /** Exception should arise on node with certain name. */
         private String gridName;
 
+        /**
+         * Default constructor.
+         */
         public BrokenAffinityFunction() {
+            // No-op.
         }
 
-        public BrokenAffinityFunction(boolean exceptionOnAllNodes, String gridName) {
-            this.exceptionOnAllNodes = exceptionOnAllNodes;
+        /**
+         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
+         * @param gridName Exception should arise on node with certain name.
+         */
+        public BrokenAffinityFunction(boolean eOnAllNodes, String gridName) {
+            this.eOnAllNodes = eOnAllNodes;
             this.gridName = gridName;
         }
 
         /** {@inheritDoc} */
         @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext affCtx) {
-            if (exceptionOnAllNodes || ignite.name().equals(gridName))
+            if (eOnAllNodes || ignite.name().equals(gridName))
                 throw new IllegalStateException("Simulated exception [locNodeId="
                     + ignite.cluster().localNode().id() + "]");
             else
@@ -377,7 +541,7 @@ public BrokenAffinityFunction(boolean exceptionOnAllNodes, String gridName) {
         /** {@inheritDoc} */
         @Override public void writeExternal(ObjectOutput out) throws IOException {
             super.writeExternal(out);
-            out.writeBoolean(exceptionOnAllNodes);
+            out.writeBoolean(eOnAllNodes);
             out.writeObject(gridName);
         }
 
@@ -385,33 +549,38 @@ public BrokenAffinityFunction(boolean exceptionOnAllNodes, String gridName) {
         @SuppressWarnings("unchecked")
         @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
             super.readExternal(in);
-            exceptionOnAllNodes = in.readBoolean();
+            eOnAllNodes = in.readBoolean();
             gridName = (String)in.readObject();
         }
     }
 
+    /**
+     * Factory that throws an exception is got created.
+     */
     private static class BrokenStoreFactory implements Factory<CacheStore<Integer, String>> {
         /** */
         @IgniteInstanceResource
         private Ignite ignite;
 
         /** Exception should arise on all nodes. */
-        boolean exceptionOnAllNodes = true;
+        boolean eOnAllNodes = true;
 
         /** Exception should arise on node with certain name. */
         public static String gridName;
 
-        public BrokenStoreFactory() {
-        }
+        /**
+         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
+         * @param gridName Exception should arise on node with certain name.
+         */
+        public BrokenStoreFactory(boolean eOnAllNodes, String gridName) {
+            this.eOnAllNodes = eOnAllNodes;
 
-        public BrokenStoreFactory(boolean exceptionOnAllNodes, String gridName) {
-            this.exceptionOnAllNodes = exceptionOnAllNodes;
-            this.gridName = gridName;
+            BrokenStoreFactory.gridName = gridName;
         }
 
         /** {@inheritDoc} */
         @Override public CacheStore<Integer, String> create() {
-            if (exceptionOnAllNodes || ignite.name().equals(gridName))
+            if (eOnAllNodes || ignite.name().equals(gridName))
                 throw new IllegalStateException("Simulated exception [locNodeId="
                     + ignite.cluster().localNode().id() + "]");
             else
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 791ed6bf9d4bc..a4ec552f562ed 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -85,6 +85,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheTxStoreValueTest;
 import org.apache.ignite.internal.processors.cache.IgniteClientCacheInitializationFailTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheFilterTest;
+import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartNoExchangeTimeoutTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartStopConcurrentTest;
@@ -228,6 +229,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheTxPreloadNoWriteTest.class);
 
         suite.addTestSuite(IgniteDynamicCacheStartSelfTest.class);
+        suite.addTestSuite(IgniteDynamicCacheStartFailSelfTest.class);
         suite.addTestSuite(IgniteDynamicCacheWithConfigStartSelfTest.class);
         suite.addTestSuite(IgniteCacheDynamicStopSelfTest.class);
         suite.addTestSuite(IgniteDynamicCacheStartStopConcurrentTest.class);

From e47a4c03bc65a55349981ad79eb6a9f7fa657a2a Mon Sep 17 00:00:00 2001
From: sk0x50 <slava.koptilin@gmail.com>
Date: Fri, 25 Aug 2017 10:48:33 +0300
Subject: [PATCH 339/516] IGNITE-1094: added cacheChangeFailureMsgSent flag

---
 .../preloader/GridDhtPartitionsExchangeFuture.java   | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index b4d02ec2bdebd..524412bf55f5f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -206,6 +206,9 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** Exchange exceptions from all participating nodes. */
     private final Map<UUID, Exception> exchangeGlobalExceptions = new ConcurrentHashMap8<>();
 
+    /** Used to track the fact that {@code DynamicCacheChangeFailureMessage} was sent. */
+    private volatile boolean cacheChangeFailureMsgSent;
+
     /** Forced Rebalance future. */
     private GridCompoundFuture<Boolean, Boolean> forcedRebFut;
 
@@ -1235,9 +1238,8 @@ public void onReceive(final ClusterNode node, final GridDhtPartitionsSingleMessa
                 log.debug("Received message for finished future (will reply only to sender) [msg=" + msg +
                     ", fut=" + this + ']');
 
-            // Custom message (DynamicCacheChangeFailureMessage) was sent.
-            // Do not need sendAllPartitions.
-            if (!exchangeGlobalExceptions.isEmpty())
+            // Custom message (DynamicCacheChangeFailureMessage) was sent. Do not need sendAllPartitions.
+            if (cacheChangeFailureMsgSent)
                 return;
 
             if (!centralizedAff)
@@ -1444,10 +1446,12 @@ private void onAllReceived(boolean discoThread) {
                     cctx.localNode(), exchId, err, cacheNames);
 
                 if (log.isDebugEnabled())
-                    log.debug("Dynamic cache change failed. Send message to all participating nodes: " + msg);
+                    log.debug("Dynamic cache change failed (send message to all participating nodes): " + msg);
 
                 cctx.discovery().sendCustomEvent(msg);
 
+                cacheChangeFailureMsgSent = true;
+
                 return;
             }
 

From d90a9207756b035294945f696fa0c00c69bdd330 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 25 Aug 2017 12:28:37 +0300
Subject: [PATCH 340/516] ignite-1094 review

---
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java   | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 524412bf55f5f..66527298efd37 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1448,10 +1448,10 @@ private void onAllReceived(boolean discoThread) {
                 if (log.isDebugEnabled())
                     log.debug("Dynamic cache change failed (send message to all participating nodes): " + msg);
 
-                cctx.discovery().sendCustomEvent(msg);
-
                 cacheChangeFailureMsgSent = true;
 
+                cctx.discovery().sendCustomEvent(msg);
+
                 return;
             }
 
@@ -1660,8 +1660,11 @@ private void updatePartitionFullMap(GridDhtPartitionsFullMessage msg) {
      * @param msg Partitions single message.
      */
     private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
-        if (msg.partitions() == null)
+        if (msg.partitions() == null) {
+            assert msg.getError() != null : msg;
+
             return;
+        }
 
         for (Map.Entry<Integer, GridDhtPartitionMap2> entry : msg.partitions().entrySet()) {
             Integer cacheId = entry.getKey();

From a7310e49948de5780845acbaada3e6ccce288265 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 25 Aug 2017 19:09:51 +0300
Subject: [PATCH 341/516] IGNITE-1094: review

---
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java       | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 66527298efd37..91ec8d250f692 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1660,11 +1660,8 @@ private void updatePartitionFullMap(GridDhtPartitionsFullMessage msg) {
      * @param msg Partitions single message.
      */
     private void updatePartitionSingleMap(GridDhtPartitionsSingleMessage msg) {
-        if (msg.partitions() == null) {
-            assert msg.getError() != null : msg;
-
+        if (msg.partitions() == null)
             return;
-        }
 
         for (Map.Entry<Integer, GridDhtPartitionMap2> entry : msg.partitions().entrySet()) {
             Integer cacheId = entry.getKey();

From 75febb824b3e261ec1c15224a024e1d4160f2f42 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Tue, 13 Jun 2017 19:41:55 +0300
Subject: [PATCH 342/516] ignite-2.1.1 Extract Ignite updates checker to
 separate class. Fixed GridUpdateNotifier test.

---
 .../cluster/GridUpdateNotifier.java           | 56 ++++++++--------
 .../cluster/HttpIgniteUpdatesChecker.java     | 65 +++++++++++++++++++
 .../cluster/GridUpdateNotifierSelfTest.java   | 10 ++-
 3 files changed, 100 insertions(+), 31 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
index 592fdd10bde73..83588c26227ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
@@ -64,8 +64,8 @@ class GridUpdateNotifier {
     /** Sleep milliseconds time for worker thread. */
     private static final int WORKER_THREAD_SLEEP_TIME = 5000;
 
-    /** Url for request version. */
-    private final static String UPDATE_NOTIFIER_URL = "https://ignite.run/update_status_ignite-plain-text.php";
+    /** Default url for request Ignite updates. */
+    private final static String DEFAULT_IGNITE_UPDATES_URL = "https://ignite.run/update_status_ignite-plain-text.php";
 
     /** Grid version. */
     private final String ver;
@@ -103,22 +103,27 @@ class GridUpdateNotifier {
     /** Worker thread to process http request. */
     private final Thread workerThread;
 
+    /** Http client for getting Ignite updates */
+    private final HttpIgniteUpdatesChecker updatesChecker;
+
     /**
      * Creates new notifier with default values.
      *
-     * @param gridName gridName
+     * @param igniteInstanceName Ignite instance name.
      * @param ver Compound Ignite version.
      * @param gw Kernal gateway.
      * @param pluginProviders Kernal gateway.
      * @param reportOnlyNew Whether or not to report only new version.
+     * @param updatesChecker Service for getting Ignite updates
      * @throws IgniteCheckedException If failed.
      */
-    GridUpdateNotifier(String gridName, String ver, GridKernalGateway gw, Collection<PluginProvider> pluginProviders,
-        boolean reportOnlyNew) throws IgniteCheckedException {
+    GridUpdateNotifier(String igniteInstanceName, String ver, GridKernalGateway gw, Collection<PluginProvider> pluginProviders,
+                       boolean reportOnlyNew, HttpIgniteUpdatesChecker updatesChecker) throws IgniteCheckedException {
         try {
             this.ver = ver;
-            this.gridName = gridName == null ? "null" : gridName;
+            this.gridName = igniteInstanceName == null ? "null" : igniteInstanceName;
             this.gw = gw;
+            this.updatesChecker = updatesChecker;
 
             SB pluginsBuilder = new SB();
 
@@ -159,6 +164,14 @@ class GridUpdateNotifier {
         }
     }
 
+    /**
+     * Creates new notifier with default Ignite updates URL
+     */
+    GridUpdateNotifier(String igniteInstanceName, String ver, GridKernalGateway gw, Collection<PluginProvider> pluginProviders,
+                       boolean reportOnlyNew) throws IgniteCheckedException {
+        this(igniteInstanceName, ver, gw, pluginProviders, reportOnlyNew, new HttpIgniteUpdatesChecker(DEFAULT_IGNITE_UPDATES_URL, CHARSET));
+    }
+
     /**
      * Gets system properties.
      *
@@ -313,34 +326,17 @@ private class UpdateChecker extends GridWorker {
                     (!F.isEmpty(vmProps) ? "&vmProps=" + encode(vmProps, CHARSET) : "") +
                         pluginsVers;
 
-                URLConnection conn = new URL(UPDATE_NOTIFIER_URL).openConnection();
-
                 if (!isCancelled()) {
-                    conn.setDoOutput(true);
-                    conn.setRequestProperty("Accept-Charset", CHARSET);
-                    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=" + CHARSET);
-
-                    conn.setConnectTimeout(3000);
-                    conn.setReadTimeout(3000);
-
                     try {
-                        try (OutputStream os = conn.getOutputStream()) {
-                            os.write(postParams.getBytes(CHARSET));
-                        }
-
-                        try (InputStream in = conn.getInputStream()) {
-                            if (in == null)
-                                return;
-
-                            BufferedReader reader = new BufferedReader(new InputStreamReader(in, CHARSET));
+                        String updatesResponse = updatesChecker.getUpdates(postParams);
 
-                            for (String line; (line = reader.readLine()) != null; ) {
-                                if (line.contains("version"))
-                                    latestVer = obtainVersionFrom(line);
-                                else if (line.contains("downloadUrl"))
-                                    downloadUrl = obtainDownloadUrlFrom(line);
-                            }
+                        String[] lines = updatesResponse.split("\n");
 
+                        for (String line : lines) {
+                            if (line.contains("version"))
+                                latestVer = obtainVersionFrom(line);
+                            else if (line.contains("downloadUrl"))
+                                downloadUrl = obtainDownloadUrlFrom(line);
                         }
                     }
                     catch (IOException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
new file mode 100644
index 0000000000000..c052c0997fe2a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
@@ -0,0 +1,65 @@
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.URL;
+import java.net.URLConnection;
+
+/**
+ * This class is responsible for getting Ignite updates information via HTTP
+ */
+public class HttpIgniteUpdatesChecker {
+    /** Url for request updates. */
+    private final String url;
+
+    /** Charset for encoding requests/responses */
+    private final String charset;
+
+    /**
+     * Creates new HTTP Ignite updates checker with following parameters
+     * @param url URL for getting Ignite updates information
+     * @param charset Charset for encoding
+     */
+    HttpIgniteUpdatesChecker(String url, String charset) {
+        this.url = url;
+        this.charset = charset;
+    }
+
+    /**
+     * Gets information about Ignite updates via HTTP
+     * @param updateRequest HTTP Request parameters
+     * @return Information about Ignite updates separated by line endings
+     * @throws IOException If HTTP request was failed
+     */
+    public String getUpdates(String updateRequest) throws IOException {
+        URLConnection conn = new URL(url).openConnection();
+        conn.setDoOutput(true);
+        conn.setRequestProperty("Accept-Charset", charset);
+        conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=" + charset);
+
+        conn.setConnectTimeout(3000);
+        conn.setReadTimeout(3000);
+
+        try (OutputStream os = conn.getOutputStream()) {
+            os.write(updateRequest.getBytes(charset));
+        }
+
+        try (InputStream in = conn.getInputStream()) {
+            if (in == null)
+                return null;
+
+            BufferedReader reader = new BufferedReader(new InputStreamReader(in, charset));
+
+            StringBuilder response = new StringBuilder();
+
+            for (String line; (line = reader.readLine()) != null; ) {
+                response.append(line).append('\n');
+            }
+
+            return response.toString();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java
index 21b91b6b3c53e..1a20f261550ba 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java
@@ -29,6 +29,8 @@
 import org.apache.ignite.plugin.PluginProvider;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
 
 /**
  * Update notifier test.
@@ -73,8 +75,14 @@ public class GridUpdateNotifierSelfTest extends GridCommonAbstractTest {
     public void testNotifier() throws Exception {
         String nodeVer = IgniteProperties.get("ignite.version");
 
+        HttpIgniteUpdatesChecker updatesCheckerMock = Mockito.mock(HttpIgniteUpdatesChecker.class);
+
+        // Return current node version and some other info
+        Mockito.when(updatesCheckerMock.getUpdates(Matchers.anyString()))
+                .thenReturn("meta=meta" + "\n" + "version=" + nodeVer + "\n" + "downloadUrl=url");
+
         GridUpdateNotifier ntf = new GridUpdateNotifier(null, nodeVer,
-            TEST_GATEWAY, Collections.<PluginProvider>emptyList(), false);
+            TEST_GATEWAY, Collections.<PluginProvider>emptyList(), false, updatesCheckerMock);
 
         ntf.checkForNewVersion(log);
 

From f813fc036802349d81d6c313c4c1846ca5ed8fd7 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Wed, 1 Feb 2017 14:21:49 +0300
Subject: [PATCH 343/516] ignite-4557 Fixed wrong affinity manager call.

---
 .../processors/cache/GridCacheAdapter.java    | 10 ++--
 .../cache/GridCacheAffinityManager.java       | 60 +++++++------------
 .../processors/cache/GridCacheContext.java    | 17 ------
 .../cache/GridCacheEvictionManager.java       |  6 +-
 .../processors/cache/GridCacheUtils.java      | 20 -------
 .../cache/affinity/GridCacheAffinityImpl.java | 16 ++---
 .../CacheDataStructuresManager.java           |  2 +-
 .../distributed/dht/GridDhtCacheAdapter.java  |  2 +-
 .../distributed/dht/GridDhtCacheEntry.java    |  2 +-
 .../dht/GridDhtLocalPartition.java            |  4 +-
 .../dht/GridDhtPartitionTopologyImpl.java     |  4 +-
 .../distributed/dht/GridDhtTxRemote.java      |  2 +-
 .../dht/GridPartitionedGetFuture.java         |  2 +-
 .../dht/GridPartitionedSingleGetFuture.java   |  2 +-
 .../dht/atomic/GridDhtAtomicCache.java        | 13 ++--
 .../GridNearAtomicSingleUpdateFuture.java     |  2 +-
 .../atomic/GridNearAtomicUpdateFuture.java    |  4 +-
 .../dht/colocated/GridDhtColocatedCache.java  |  6 +-
 .../colocated/GridDhtColocatedLockFuture.java |  4 +-
 .../preloader/GridDhtPartitionDemander.java   |  8 +--
 .../preloader/GridDhtPartitionSupplier.java   | 12 ++--
 .../dht/preloader/GridDhtPreloader.java       |  4 +-
 .../distributed/near/GridNearAtomicCache.java |  2 +-
 .../distributed/near/GridNearCacheEntry.java  |  6 +-
 .../distributed/near/GridNearGetFuture.java   |  4 +-
 .../distributed/near/GridNearLockFuture.java  |  2 +-
 ...OptimisticSerializableTxPrepareFuture.java |  2 +-
 .../GridNearOptimisticTxPrepareFuture.java    |  2 +-
 .../GridNearPessimisticTxPrepareFuture.java   |  2 +-
 .../near/GridNearTransactionalCache.java      |  6 +-
 .../near/GridNearTxFinishFuture.java          |  2 +-
 .../cache/query/GridCacheQueryManager.java    |  9 +--
 .../CacheContinuousQueryHandler.java          |  1 -
 .../CacheContinuousQueryManager.java          |  2 +-
 .../cache/transactions/IgniteTxAdapter.java   |  4 +-
 .../transactions/IgniteTxLocalAdapter.java    |  4 +-
 .../transactions/TxDeadlockDetection.java     |  2 +-
 .../datastreamer/DataStreamerImpl.java        |  2 +-
 .../datastructures/GridCacheSetImpl.java      |  2 +-
 .../datastructures/GridSetQueryPredicate.java |  2 +-
 .../processors/job/GridJobProcessor.java      |  2 +-
 .../cache/CacheAffinityCallSelfTest.java      |  4 +-
 .../GridCacheAbstractFullApiSelfTest.java     |  6 +-
 ...gniteCacheConfigVariationsFullApiTest.java |  6 +-
 .../IgniteCachePeekModesAbstractTest.java     |  8 +--
 ...tQueueFailoverDataConsistencySelfTest.java |  2 +-
 ...teCacheClientNodeChangingTopologyTest.java |  8 +--
 .../TxOptimisticDeadlockDetectionTest.java    |  2 +-
 .../TxPessimisticDeadlockDetectionTest.java   |  2 +-
 .../processors/query/h2/IgniteH2Indexing.java |  2 +-
 .../query/h2/opt/GridH2IndexBase.java         |  2 +-
 51 files changed, 126 insertions(+), 176 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 7aa48199bb28a..9b1e0cc7721c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -778,7 +778,7 @@ public String toString() {
                 boolean nearKey;
 
                 if (!(modes.near && modes.primary && modes.backup)) {
-                    boolean keyPrimary = ctx.affinity().primary(ctx.localNode(), part, topVer);
+                    boolean keyPrimary = ctx.affinity().primaryByPartition(ctx.localNode(), part, topVer);
 
                     if (keyPrimary) {
                         if (!modes.primary)
@@ -787,7 +787,7 @@ public String toString() {
                         nearKey = false;
                     }
                     else {
-                        boolean keyBackup = ctx.affinity().belongs(ctx.localNode(), part, topVer);
+                        boolean keyBackup = ctx.affinity().partitionBelongs(ctx.localNode(), part, topVer);
 
                         if (keyBackup) {
                             if (!modes.backup)
@@ -808,7 +808,7 @@ public String toString() {
                     }
                 }
                 else {
-                    nearKey = !ctx.affinity().belongs(ctx.localNode(), part, topVer);
+                    nearKey = !ctx.affinity().partitionBelongs(ctx.localNode(), part, topVer);
 
                     if (nearKey) {
                         // Swap and offheap are disabled for near cache.
@@ -3813,8 +3813,8 @@ IgniteInternalFuture<?> globalLoadCacheAsync(@Nullable IgniteBiPredicate<K, V> p
             /** {@inheritDoc} */
             @Override public boolean apply(ClusterNode clusterNode) {
                 return clusterNode.version().compareTo(PartitionSizeLongTask.SINCE_VER) >= 0 &&
-                    ((modes.primary && aff.primary(clusterNode, part, topVer)) ||
-                        (modes.backup && aff.backup(clusterNode, part, topVer)));
+                    ((modes.primary && aff.primaryByPartition(clusterNode, part, topVer)) ||
+                        (modes.backup && aff.backupByPartition(clusterNode, part, topVer)));
             }
         }).nodes();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
index 8b7be1b3288e2..d85e76e2bbd82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
@@ -238,8 +238,8 @@ public Object affinityKey(Object key) {
      * @param topVer Topology version.
      * @return Affinity nodes.
      */
-    public List<ClusterNode> nodes(Object key, AffinityTopologyVersion topVer) {
-        return nodes(partition(key), topVer);
+    public List<ClusterNode> nodesByKey(Object key, AffinityTopologyVersion topVer) {
+        return nodesByPartition(partition(key), topVer);
     }
 
     /**
@@ -247,7 +247,7 @@ public List<ClusterNode> nodes(Object key, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Affinity nodes.
      */
-    public List<ClusterNode> nodes(int part, AffinityTopologyVersion topVer) {
+    public List<ClusterNode> nodesByPartition(int part, AffinityTopologyVersion topVer) {
         if (cctx.isLocal())
             topVer = LOC_CACHE_TOP_VER;
 
@@ -282,8 +282,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Primary node for given key.
      */
-    @Nullable public ClusterNode primary(Object key, AffinityTopologyVersion topVer) {
-        return primary(partition(key), topVer);
+    @Nullable public ClusterNode primaryByKey(Object key, AffinityTopologyVersion topVer) {
+        return primaryByPartition(partition(key), topVer);
     }
 
     /**
@@ -291,8 +291,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Primary node for given key.
      */
-    @Nullable public ClusterNode primary(int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    @Nullable public ClusterNode primaryByPartition(int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         if (nodes.isEmpty())
             return null;
@@ -306,8 +306,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code True} if checked node is primary for given key.
      */
-    public boolean primary(ClusterNode n, Object key, AffinityTopologyVersion topVer) {
-        return F.eq(primary(key, topVer), n);
+    public boolean primaryByKey(ClusterNode n, Object key, AffinityTopologyVersion topVer) {
+        return F.eq(primaryByKey(key, topVer), n);
     }
 
     /**
@@ -316,8 +316,8 @@ public boolean primary(ClusterNode n, Object key, AffinityTopologyVersion topVer
      * @param topVer Topology version.
      * @return {@code True} if checked node is primary for given partition.
      */
-    public boolean primary(ClusterNode n, int part, AffinityTopologyVersion topVer) {
-        return F.eq(primary(part, topVer), n);
+    public boolean primaryByPartition(ClusterNode n, int part, AffinityTopologyVersion topVer) {
+        return F.eq(primaryByPartition(part, topVer), n);
     }
 
     /**
@@ -325,8 +325,8 @@ public boolean primary(ClusterNode n, int part, AffinityTopologyVersion topVer)
      * @param topVer Topology version.
      * @return Backup nodes.
      */
-    public Collection<ClusterNode> backups(Object key, AffinityTopologyVersion topVer) {
-        return backups(partition(key), topVer);
+    public Collection<ClusterNode> backupsByKey(Object key, AffinityTopologyVersion topVer) {
+        return backupsByPartition(partition(key), topVer);
     }
 
     /**
@@ -334,8 +334,8 @@ public Collection<ClusterNode> backups(Object key, AffinityTopologyVersion topVe
      * @param topVer Topology version.
      * @return Backup nodes.
      */
-    public Collection<ClusterNode> backups(int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    private Collection<ClusterNode> backupsByPartition(int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         assert !F.isEmpty(nodes);
 
@@ -351,35 +351,21 @@ public Collection<ClusterNode> backups(int part, AffinityTopologyVersion topVer)
      * @param topVer Topology version.
      * @return {@code True} if checked node is a backup node for given partition.
      */
-    public boolean backup(ClusterNode n, int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    public boolean backupByPartition(ClusterNode n, int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         assert !F.isEmpty(nodes);
 
         return nodes.indexOf(n) > 0;
     }
 
-    /**
-     * @param keys keys.
-     * @param topVer Topology version.
-     * @return Nodes for the keys.
-     */
-    public Collection<ClusterNode> remoteNodes(Iterable keys, AffinityTopologyVersion topVer) {
-        Collection<Collection<ClusterNode>> colcol = new GridLeanSet<>();
-
-        for (Object key : keys)
-            colcol.add(nodes(key, topVer));
-
-        return F.view(F.flatCollections(colcol), F.remoteNodes(cctx.localNodeId()));
-    }
-
     /**
      * @param key Key to check.
      * @param topVer Topology version.
      * @return {@code true} if given key belongs to local node.
      */
-    public boolean localNode(Object key, AffinityTopologyVersion topVer) {
-        return localNode(partition(key), topVer);
+    public boolean keyLocalNode(Object key, AffinityTopologyVersion topVer) {
+        return partitionLocalNode(partition(key), topVer);
     }
 
     /**
@@ -387,10 +373,10 @@ public boolean localNode(Object key, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code true} if given partition belongs to local node.
      */
-    public boolean localNode(int part, AffinityTopologyVersion topVer) {
+    public boolean partitionLocalNode(int part, AffinityTopologyVersion topVer) {
         assert part >= 0 : "Invalid partition: " + part;
 
-        return nodes(part, topVer).contains(cctx.localNode());
+        return nodesByPartition(part, topVer).contains(cctx.localNode());
     }
 
     /**
@@ -399,11 +385,11 @@ public boolean localNode(int part, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code true} if given partition belongs to specified node.
      */
-    public boolean belongs(ClusterNode node, int part, AffinityTopologyVersion topVer) {
+    public boolean partitionBelongs(ClusterNode node, int part, AffinityTopologyVersion topVer) {
         assert node != null;
         assert part >= 0 : "Invalid partition: " + part;
 
-        return nodes(part, topVer).contains(node);
+        return nodesByPartition(part, topVer).contains(node);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 6322f9f03550a..3b44b5096f0b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -1589,23 +1589,6 @@ private void map(GridDhtCacheEntry entry, Iterable<ClusterNode> nodes,
         }
     }
 
-    /**
-     * Checks if at least one of the given keys belongs to one of the given partitions.
-     *
-     * @param keys Collection of keys to check.
-     * @param movingParts Collection of partitions to check against.
-     * @return {@code True} if there exist a key in collection {@code keys} that belongs
-     *      to one of partitions in {@code movingParts}
-     */
-    public boolean hasKey(Iterable<? extends K> keys, Collection<Integer> movingParts) {
-        for (K key : keys) {
-            if (movingParts.contains(affinity().partition(key)))
-                return true;
-        }
-
-        return false;
-    }
-
     /**
      * Check whether conflict resolution is required.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
index 134e743d4fd6e..f8722d6a4f338 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
@@ -808,7 +808,7 @@ public void touch(GridCacheEntryEx e, AffinityTopologyVersion topVer) {
             return;
 
         // Don't track non-primary entries if evicts are synchronized.
-        if (!cctx.isNear() && evictSync && !cctx.affinity().primary(cctx.localNode(), e.partition(), topVer))
+        if (!cctx.isNear() && evictSync && !cctx.affinity().primaryByPartition(cctx.localNode(), e.partition(), topVer))
             return;
 
         if (!busyLock.enterBusy())
@@ -910,7 +910,7 @@ public boolean evict(@Nullable GridCacheEntryEx entry, @Nullable GridCacheVersio
         if (evictSyncAgr) {
             assert !cctx.isNear(); // Make sure cache is not NEAR.
 
-            if (cctx.affinity().backups(
+            if (cctx.affinity().backupsByKey(
                     entry.key(),
                     cctx.topology().topologyVersion()).contains(cctx.localNode()) &&
                 evictSync)
@@ -1498,7 +1498,7 @@ void addEvent(DiscoveryEvent evt) {
                         if (!evts.isEmpty())
                             break;
 
-                        if (!cctx.affinity().primary(loc, it.next(), topVer))
+                        if (!cctx.affinity().primaryByPartition(loc, it.next(), topVer))
                             it.remove();
                     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 8ee77e36c3a77..5c21d7e4a8e98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -916,26 +916,6 @@ public static <K, V> void unwindEvicts(GridCacheSharedContext<K, V> ctx) {
             unwindEvicts(cacheCtx);
     }
 
-    /**
-     * Gets primary node on which given key is cached.
-     *
-     * @param ctx Cache.
-     * @param key Key to find primary node for.
-     * @return Primary node for the key.
-     */
-    @SuppressWarnings( {"unchecked"})
-    @Nullable public static ClusterNode primaryNode(GridCacheContext ctx, Object key) {
-        assert ctx != null;
-        assert key != null;
-
-        CacheConfiguration cfg = ctx.cache().configuration();
-
-        if (cfg.getCacheMode() != PARTITIONED)
-            return ctx.localNode();
-
-        return ctx.affinity().primary(key, ctx.affinity().affinityTopologyVersion());
-    }
-
     /**
      * @param asc {@code True} for ascending.
      * @return Descending order comparator.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
index 9e85bad610432..11361a27d384a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
@@ -82,21 +82,21 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public boolean isPrimary(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().primary(n, key, topologyVersion());
+        return cctx.affinity().primaryByKey(n, key, topologyVersion());
     }
 
     /** {@inheritDoc} */
     @Override public boolean isBackup(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().backups(key, topologyVersion()).contains(n);
+        return cctx.affinity().backupsByKey(key, topologyVersion()).contains(n);
     }
 
     /** {@inheritDoc} */
     @Override public boolean isPrimaryOrBackup(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().belongs(n, cctx.affinity().partition(key), topologyVersion());
+        return cctx.affinity().partitionBelongs(n, cctx.affinity().partition(key), topologyVersion());
     }
 
     /** {@inheritDoc} */
@@ -126,7 +126,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
         AffinityTopologyVersion topVer = topologyVersion();
 
         for (int partsCnt = partitions(), part = 0; part < partsCnt; part++) {
-            for (ClusterNode affNode : cctx.affinity().nodes(part, topVer)) {
+            for (ClusterNode affNode : cctx.affinity().nodesByPartition(part, topVer)) {
                 if (n.id().equals(affNode.id())) {
                     parts.add(part);
 
@@ -142,7 +142,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public ClusterNode mapPartitionToNode(int part) {
         A.ensure(part >= 0 && part < partitions(), "part >= 0 && part < total partitions");
 
-        return F.first(cctx.affinity().nodes(part, topologyVersion()));
+        return F.first(cctx.affinity().nodesByPartition(part, topologyVersion()));
     }
 
     /** {@inheritDoc} */
@@ -204,7 +204,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
         Map<ClusterNode, Collection<K>> res = new HashMap<>(nodesCnt, 1.0f);
 
         for (K key : keys) {
-            ClusterNode primary = cctx.affinity().primary(key, topVer);
+            ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
             if (primary == null)
                 throw new IgniteException("Failed to get primary node [topVer=" + topVer + ", key=" + key + ']');
@@ -227,14 +227,14 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public Collection<ClusterNode> mapKeyToPrimaryAndBackups(K key) {
         A.notNull(key, "key");
 
-        return cctx.affinity().nodes(partition(key), topologyVersion());
+        return cctx.affinity().nodesByPartition(partition(key), topologyVersion());
     }
 
     /** {@inheritDoc} */
     @Override public Collection<ClusterNode> mapPartitionToPrimaryAndBackups(int part) {
         A.ensure(part >= 0 && part < partitions(), "part >= 0 && part < total partitions");
 
-        return cctx.affinity().nodes(part, topologyVersion());
+        return cctx.affinity().nodesByPartition(part, topologyVersion());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index 366a4a920b9ee..2b3080981ec36 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -455,7 +455,7 @@ private void removeSetData(IgniteUuid setId, AffinityTopologyVersion topVer) thr
         Collection<SetItemKey> keys = new ArrayList<>(BATCH_SIZE);
 
         for (SetItemKey key : set) {
-            if (!loc && !aff.primary(cctx.localNode(), key, topVer))
+            if (!loc && !aff.primaryByKey(cctx.localNode(), key, topVer))
                 continue;
 
             keys.add(key);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index be7fa5525deba..2fa934b79db71 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -933,7 +933,7 @@ public void sendTtlUpdateRequest(@Nullable final IgniteCacheExpiryPolicy expiryP
                     AffinityTopologyVersion topVer = ctx.shared().exchange().readyAffinityVersion();
 
                     for (Map.Entry<KeyCacheObject, GridCacheVersion> e : entries.entrySet()) {
-                        List<ClusterNode> nodes = ctx.affinity().nodes(e.getKey(), topVer);
+                        List<ClusterNode> nodes = ctx.affinity().nodesByKey(e.getKey(), topVer);
 
                         for (int i = 0; i < nodes.size(); i++) {
                             ClusterNode node = nodes.get(i);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
index cf4085ba02335..39571ff4720e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
@@ -402,7 +402,7 @@ public Collection<UUID> readers() throws GridCacheEntryRemovedException {
         }
 
         // If remote node is (primary?) or back up, don't add it as a reader.
-        if (cctx.affinity().belongs(node, partition(), topVer)) {
+        if (cctx.affinity().partitionBelongs(node, partition(), topVer)) {
             if (log.isDebugEnabled())
                 log.debug("Ignoring near reader because remote node is affinity node [locNodeId=" + cctx.localNodeId()
                     + ", rmtNodeId=" + nodeId + ", key=" + key + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 668a1cdc006d9..b21463843bbcf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -614,7 +614,7 @@ void onUnlock() {
      * @return {@code True} if local node is primary for this partition.
      */
     public boolean primary(AffinityTopologyVersion topVer) {
-        return cctx.affinity().primary(cctx.localNode(), id, topVer);
+        return cctx.affinity().primaryByPartition(cctx.localNode(), id, topVer);
     }
 
     /**
@@ -622,7 +622,7 @@ public boolean primary(AffinityTopologyVersion topVer) {
      * @return {@code True} if local node is backup for this partition.
      */
     public boolean backup(AffinityTopologyVersion topVer) {
-        return cctx.affinity().backup(cctx.localNode(), id, topVer);
+        return cctx.affinity().backupByPartition(cctx.localNode(), id, topVer);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 1ad0ff0538fa5..23485f985767d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -568,7 +568,7 @@ else if (!node2part.nodeId().equals(loc.id())) {
             for (int p = 0; p < num; p++) {
                 GridDhtLocalPartition locPart = localPartition(p, topVer, false, false);
 
-                if (cctx.affinity().localNode(p, topVer)) {
+                if (cctx.affinity().partitionLocalNode(p, topVer)) {
                     // This partition will be created during next topology event,
                     // which obviously has not happened at this point.
                     if (locPart == null) {
@@ -691,7 +691,7 @@ private GridDhtLocalPartition localPartition(int p,
         try {
             loc = locParts.get(p);
 
-            boolean belongs = cctx.affinity().localNode(p, topVer);
+            boolean belongs = cctx.affinity().partitionLocalNode(p, topVer);
 
             if (loc != null && loc.state() == EVICTED) {
                 locParts.set(p, loc = null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
index 8942ef9d9178c..399736e8f4278 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
@@ -286,7 +286,7 @@ IgniteUuid remoteFutureId() {
             return true;
 
         // Check if we are on the backup node.
-        return !cacheCtx.affinity().backups(key, topVer).contains(cctx.localNode());
+        return !cacheCtx.affinity().backupsByKey(key, topVer).contains(cctx.localNode());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index c41711c245028..519239aecb532 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -380,7 +380,7 @@ private boolean map(
     ) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index 2b5624b489b6f..a3f6b72376e8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -325,7 +325,7 @@ private void map(AffinityTopologyVersion topVer) {
     @Nullable private ClusterNode mapKeyToNode(AffinityTopologyVersion topVer) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index dc6d3dd8271dd..16e51ee0c2815 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -2469,7 +2469,7 @@ private UpdateSingleResult updateSingle(
 
                 assert !(newConflictVer instanceof GridCacheVersionEx) : newConflictVer;
 
-                boolean primary = !req.fastMap() || ctx.affinity().primary(ctx.localNode(), entry.partition(),
+                boolean primary = !req.fastMap() || ctx.affinity().primaryByPartition(ctx.localNode(), entry.partition(),
                     req.topologyVersion());
 
                 Object writeVal = op == TRANSFORM ? req.entryProcessor(i) : req.writeValue(i);
@@ -2559,7 +2559,7 @@ else if (conflictCtx.isMerge())
 
                 if (hasNear) {
                     if (primary && updRes.sendToDht()) {
-                        if (!ctx.affinity().belongs(node, entry.partition(), topVer)) {
+                        if (!ctx.affinity().partitionBelongs(node, entry.partition(), topVer)) {
                             // If put the same value as in request then do not need to send it back.
                             if (op == TRANSFORM || writeVal != updRes.newValue()) {
                                 res.addNearValue(i,
@@ -2690,7 +2690,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
                 Map<KeyCacheObject, CacheObject> storeMap = req.fastMap() ?
                     F.view(putMap, new P1<CacheObject>() {
                         @Override public boolean apply(CacheObject key) {
-                            return ctx.affinity().primary(ctx.localNode(), key, req.topologyVersion());
+                            return ctx.affinity().primaryByKey(ctx.localNode(), key, req.topologyVersion());
                         }
                     }) :
                     putMap;
@@ -2713,7 +2713,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
                 Collection<KeyCacheObject> storeKeys = req.fastMap() ?
                     F.view(rmvKeys, new P1<Object>() {
                         @Override public boolean apply(Object key) {
-                            return ctx.affinity().primary(ctx.localNode(), key, req.topologyVersion());
+                            return ctx.affinity().primaryByKey(ctx.localNode(), key, req.topologyVersion());
                         }
                     }) :
                     rmvKeys;
@@ -2752,7 +2752,8 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
 
                     assert writeVal != null || op == DELETE : "null write value found.";
 
-                    boolean primary = !req.fastMap() || ctx.affinity().primary(ctx.localNode(), entry.key(),
+                    boolean primary = !req.fastMap() || ctx.affinity().primaryByPartition(ctx.localNode(),
+                        entry.partition(),
                         req.topologyVersion());
 
                     Collection<UUID> readers = null;
@@ -2848,7 +2849,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
 
                     if (hasNear) {
                         if (primary) {
-                            if (!ctx.affinity().belongs(node, entry.partition(), topVer)) {
+                            if (!ctx.affinity().partitionBelongs(node, entry.partition(), topVer)) {
                                 int idx = firstEntryIdx + i;
 
                                 if (req.operation() == TRANSFORM) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
index 7376affca4455..891a20c2884a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
@@ -543,7 +543,7 @@ private GridNearAtomicAbstractUpdateRequest mapSingleUpdate(AffinityTopologyVers
         else
             val = EntryProcessorResourceInjectorProxy.wrap(cctx.kernalContext(), (EntryProcessor)val);
 
-        ClusterNode primary = cctx.affinity().primary(cacheKey, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(cacheKey, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to map keys for cache (all partition nodes " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index a252d9ae30c2b..9bdd1becb7600 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -937,7 +937,7 @@ else if (conflictRmvVals != null) {
         else
             val = EntryProcessorResourceInjectorProxy.wrap(cctx.kernalContext(), (EntryProcessor)val);
 
-        ClusterNode primary = cctx.affinity().primary(cacheKey.partition(), topVer);
+        ClusterNode primary = cctx.affinity().primaryByPartition(cacheKey.partition(), topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to map keys for cache (all partition nodes " +
@@ -988,7 +988,7 @@ private List<ClusterNode> mapKey(KeyCacheObject key, AffinityTopologyVersion top
 
         // If we can send updates in parallel - do it.
         return fastMap ? cctx.topology().nodes(affMgr.partition(key), topVer) :
-            Collections.singletonList(affMgr.primary(key, topVer));
+            Collections.singletonList(affMgr.primaryByKey(key, topVer));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index c9fc983d4850c..e1e0ec2bf087e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -174,7 +174,7 @@ public GridDistributedCacheEntry entryExx(
         AffinityTopologyVersion topVer,
         boolean allowDetached
     ) {
-        return allowDetached && !ctx.affinity().primary(ctx.localNode(), key, topVer) ?
+        return allowDetached && !ctx.affinity().primaryByKey(ctx.localNode(), key, topVer) ?
             createEntry(key) : entryExx(key, topVer);
     }
 
@@ -670,7 +670,7 @@ else if (!skipVals && ctx.config().isStatisticsEnabled())
                     assert topVer.compareTo(AffinityTopologyVersion.ZERO) > 0;
 
                     // Send request to remove from remote nodes.
-                    ClusterNode primary = ctx.affinity().primary(key, topVer);
+                    ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                     if (primary == null) {
                         if (log.isDebugEnabled())
@@ -790,7 +790,7 @@ public void removeLocks(long threadId, GridCacheVersion ver, Collection<KeyCache
                         map = U.newHashMap(affNodes.size());
                     }
 
-                    ClusterNode primary = ctx.affinity().primary(key, topVer);
+                    ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                     if (primary == null) {
                         if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index 40e87ee2cfd7c..b373818d1f32f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -1201,7 +1201,7 @@ private boolean mapAsPrimary(Collection<KeyCacheObject> keys, AffinityTopologyVe
         boolean explicit = false;
 
         for (KeyCacheObject key : keys) {
-            if (!cctx.affinity().primary(cctx.localNode(), key, topVer)) {
+            if (!cctx.affinity().primaryByKey(cctx.localNode(), key, topVer)) {
                 // Remove explicit locks added so far.
                 for (KeyCacheObject k : keys)
                     cctx.mvcc().removeExplicitLock(threadId, cctx.txKey(k), lockVer);
@@ -1285,7 +1285,7 @@ private GridNearLockMapping map(
     ) throws IgniteCheckedException {
         assert mapping == null || mapping.node() != null;
 
-        ClusterNode primary = cctx.affinity().primary(key, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to lock keys " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index daae1e2aa144a..8586b69b5961d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -598,7 +598,7 @@ public void handleSupplyMessage(
             for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
                 int p = e.getKey();
 
-                if (cctx.affinity().localNode(p, topVer)) {
+                if (cctx.affinity().partitionLocalNode(p, topVer)) {
                     GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                     assert part != null;
@@ -668,7 +668,7 @@ public void handleSupplyMessage(
 
             // Only request partitions based on latest topology version.
             for (Integer miss : supply.missed()) {
-                if (cctx.affinity().localNode(miss, topVer))
+                if (cctx.affinity().partitionLocalNode(miss, topVer))
                     fut.partitionMissed(id, miss);
             }
 
@@ -1359,7 +1359,7 @@ private void demandFromNode(
                         for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
                             int p = e.getKey();
 
-                            if (cctx.affinity().localNode(p, topVer)) {
+                            if (cctx.affinity().partitionLocalNode(p, topVer)) {
                                 GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                                 assert part != null;
@@ -1436,7 +1436,7 @@ private void demandFromNode(
 
                         // Only request partitions based on latest topology version.
                         for (Integer miss : s.supply().missed()) {
-                            if (cctx.affinity().localNode(miss, topVer))
+                            if (cctx.affinity().partitionLocalNode(miss, topVer))
                                 fut.partitionMissed(node.id(), miss);
                         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index b082c4736cb9c..994242354c05d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -302,7 +302,7 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
                             (Iterator<GridCacheMapEntry>)sctx.entryIt : loc.allEntries().iterator();
 
                         while (entIt.hasNext()) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition, so we send '-1' partition and move on.
                                 s.missed(part);
 
@@ -387,7 +387,7 @@ else if (log.isDebugEnabled())
                             boolean prepared = false;
 
                             while (iter.hasNext()) {
-                                if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                                if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                     // Demander no longer needs this partition,
                                     // so we send '-1' partition and move on.
                                     s.missed(part);
@@ -510,7 +510,7 @@ else if (log.isDebugEnabled())
                             (Iterator<GridCacheEntryInfo>)sctx.entryIt : entries.iterator();
 
                         while (lsnrIt.hasNext()) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition,
                                 // so we send '-1' partition and move on.
                                 s.missed(part);
@@ -808,7 +808,7 @@ private void processOldDemandMessage(GridDhtPartitionDemandMessage d, UUID id) {
                     boolean partMissing = false;
 
                     for (GridCacheEntryEx e : loc.allEntries()) {
-                        if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                        if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                             // Demander no longer needs this partition, so we send '-1' partition and move on.
                             s.missed(part);
 
@@ -859,7 +859,7 @@ else if (log.isDebugEnabled())
                                 boolean prepared = false;
 
                                 for (Map.Entry<byte[], GridCacheSwapEntry> e : iter) {
-                                    if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                                    if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                         // Demander no longer needs this partition,
                                         // so we send '-1' partition and move on.
                                         s.missed(part);
@@ -947,7 +947,7 @@ else if (log.isDebugEnabled())
                         swapLsnr = null;
 
                         for (GridCacheEntryInfo info : entries) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition,
                                 // so we send '-1' partition and move on.
                                 s.missed(part);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 692e7c0de45a6..c012e452220ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -289,7 +289,7 @@ private IgniteCheckedException stopError() {
             }
 
             // If partition belongs to local node.
-            if (cctx.affinity().localNode(p, topVer)) {
+            if (cctx.affinity().partitionLocalNode(p, topVer)) {
                 GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                 assert part != null;
@@ -349,7 +349,7 @@ private IgniteCheckedException stopError() {
      * @return Picked owners.
      */
     private Collection<ClusterNode> pickedOwners(int p, AffinityTopologyVersion topVer) {
-        Collection<ClusterNode> affNodes = cctx.affinity().nodes(p, topVer);
+        Collection<ClusterNode> affNodes = cctx.affinity().nodesByPartition(p, topVer);
 
         int affCnt = affNodes.size();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index b843e4e6d3db1..41632ef338b58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -161,7 +161,7 @@ public void processNearAtomicUpdateResponse(
             if (F.contains(failed, key))
                 continue;
 
-            if (ctx.affinity().belongs(ctx.localNode(), ctx.affinity().partition(key), req.topologyVersion())) { // Reader became backup.
+            if (ctx.affinity().partitionBelongs(ctx.localNode(), ctx.affinity().partition(key), req.topologyVersion())) { // Reader became backup.
                 GridCacheEntryEx entry = peekEx(key);
 
                 if (entry != null && entry.markObsolete(ver))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index 30fc213fc63e2..d022805a76bf2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -112,7 +112,7 @@ public GridNearCacheEntry(
                 return false;
             }
 
-            if (cctx.affinity().backup(cctx.localNode(), part, topVer)) {
+            if (cctx.affinity().backupByPartition(cctx.localNode(), part, topVer)) {
                 this.topVer = AffinityTopologyVersion.NONE;
 
                 return false;
@@ -162,7 +162,7 @@ public void initializeFromDht(AffinityTopologyVersion topVer) throws GridCacheEn
                                 }
                             }
 
-                            ClusterNode primaryNode = cctx.affinity().primary(key, topVer);
+                            ClusterNode primaryNode = cctx.affinity().primaryByKey(key, topVer);
 
                             if (primaryNode == null)
                                 this.topVer = AffinityTopologyVersion.NONE;
@@ -686,7 +686,7 @@ private void primaryNode(UUID nodeId, AffinityTopologyVersion topVer) {
         ClusterNode primary = null;
 
         try {
-            primary = cctx.affinity().primary(part, topVer);
+            primary = cctx.affinity().primaryByPartition(part, topVer);
         }
         catch (IllegalStateException ignore) {
             // Do not have affinity history.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index cb47498be0167..fb2843c02eec9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -413,7 +413,7 @@ private Map<KeyCacheObject, GridNearCacheEntry> map(
     ) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
@@ -726,7 +726,7 @@ private Map<K, V> loadEntries(
                     info.unmarshalValue(cctx, cctx.deploy().globalLoader());
 
                     // Entries available locally in DHT should not be loaded into near cache for reading.
-                    if (!cctx.affinity().localNode(info.key(), cctx.affinity().affinityTopologyVersion())) {
+                    if (!cctx.affinity().keyLocalNode(info.key(), cctx.affinity().affinityTopologyVersion())) {
                         GridNearCacheEntry entry = savedEntries.get(info.key());
 
                         if (entry == null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index 491b4ece0ea13..8035655237635 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -1371,7 +1371,7 @@ private GridNearLockMapping map(
     ) throws IgniteCheckedException {
         assert mapping == null || mapping.node() != null;
 
-        ClusterNode primary = cctx.affinity().primary(key, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to lock keys " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index 4cbfb27d34857..f6ca77a1ef96e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -527,7 +527,7 @@ private void map(
         GridCacheContext cacheCtx = entry.context();
 
         List<ClusterNode> nodes = cacheCtx.isLocal() ?
-            cacheCtx.affinity().nodes(entry.key(), topVer) :
+            cacheCtx.affinity().nodesByKey(entry.key(), topVer) :
             cacheCtx.topology().nodes(cacheCtx.affinity().partition(entry.key()), topVer);
 
         txMapping.addMapping(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 91cfbda6e15d3..7e446f3baa0d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -601,7 +601,7 @@ private GridDistributedTxMapping map(
             nodes = cacheCtx.topology().nodes(cached0.partition(), topVer);
         else
             nodes = cacheCtx.isLocal() ?
-                cacheCtx.affinity().nodes(entry.key(), topVer) :
+                cacheCtx.affinity().nodesByKey(entry.key(), topVer) :
                 cacheCtx.topology().nodes(cacheCtx.affinity().partition(entry.key()), topVer);
 
         txMapping.addMapping(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index 5c09398b4ecb1..28cd3584362cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -194,7 +194,7 @@ private void preparePessimistic() {
             GridCacheContext cacheCtx = txEntry.context();
 
             List<ClusterNode> nodes = cacheCtx.isLocal() ?
-                cacheCtx.affinity().nodes(txEntry.key(), topVer) :
+                cacheCtx.affinity().nodesByKey(txEntry.key(), topVer) :
                 cacheCtx.topology().nodes(cacheCtx.affinity().partition(txEntry.key()), topVer);
 
             ClusterNode primary = F.first(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
index b3eb7551f05a0..940dd809779a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
@@ -476,7 +476,7 @@ private void processLockResponse(UUID nodeId, GridNearLockResponse res) {
      * @return {@code True} if entry is locally mapped as a primary or back up node.
      */
     protected boolean isNearLocallyMapped(GridCacheEntryEx e, AffinityTopologyVersion topVer) {
-        return ctx.affinity().belongs(ctx.localNode(), e.partition(), topVer);
+        return ctx.affinity().partitionBelongs(ctx.localNode(), e.partition(), topVer);
     }
 
     /**
@@ -548,7 +548,7 @@ protected boolean evictNearEntry(GridCacheEntryEx e, GridCacheVersion obsoleteVe
                             topVer = cand.topologyVersion();
 
                             // Send request to remove from remote nodes.
-                            ClusterNode primary = ctx.affinity().primary(key, topVer);
+                            ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                             if (primary == null) {
                                 if (log.isDebugEnabled())
@@ -668,7 +668,7 @@ public void removeLocks(GridCacheVersion ver, Collection<KeyCacheObject> keys) {
                                     map = U.newHashMap(affNodes.size());
                                 }
 
-                                ClusterNode primary = ctx.affinity().primary(key, cand.topologyVersion());
+                                ClusterNode primary = ctx.affinity().primaryByKey(key, cand.topologyVersion());
 
                                 if (primary == null) {
                                     if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 46604c7524527..6d3f21f095865 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -358,7 +358,7 @@ else if (tx.implicit() && tx.isSystemInvalidate()) { // Finish implicit transact
                             GridCacheContext cacheCtx = e.context();
 
                             try {
-                                if (e.op() != NOOP && !cacheCtx.affinity().localNode(e.key(), topVer)) {
+                                if (e.op() != NOOP && !cacheCtx.affinity().keyLocalNode(e.key(), topVer)) {
                                     GridCacheEntryEx entry = cacheCtx.cache().peekEx(e.key());
 
                                     if (entry != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 7efb746d4428e..47f1bed98680c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -1580,11 +1580,11 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                     // Other types are filtered in indexing manager.
                     if (!cctx.isReplicated() && qry.type() == SCAN && qry.partition() == null &&
                         cctx.config().getCacheMode() != LOCAL && !incBackups &&
-                        !cctx.affinity().primary(cctx.localNode(), key, topVer)) {
+                        !cctx.affinity().primaryByKey(cctx.localNode(), key, topVer)) {
                         if (log.isDebugEnabled())
                             log.debug("Ignoring backup element [row=" + row +
                                 ", cacheMode=" + cctx.config().getCacheMode() + ", incBackups=" + incBackups +
-                                ", primary=" + cctx.affinity().primary(cctx.localNode(), key, topVer) + ']');
+                                ", primary=" + cctx.affinity().primaryByKey(cctx.localNode(), key, topVer) + ']');
 
                         continue;
                     }
@@ -1592,7 +1592,8 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                     V val = row.getValue();
 
                     if (log.isDebugEnabled()) {
-                        ClusterNode primaryNode = CU.primaryNode(cctx, key);
+                        ClusterNode primaryNode = cctx.affinity().primaryByKey(key,
+                            cctx.affinity().affinityTopologyVersion());
 
                         log.debug(S.toString("Record",
                             "key", key, true,
@@ -2355,7 +2356,7 @@ public Collection<GridCacheSqlMetadata> sqlMetadata() throws IgniteCheckedExcept
 
                 return new IgniteBiPredicate<K, V>() {
                     @Override public boolean apply(K k, V v) {
-                        return cache.context().affinity().primary(ctx.discovery().localNode(), k, NONE);
+                        return cache.context().affinity().primaryByKey(ctx.discovery().localNode(), k, NONE);
                     }
                 };
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 165b8b77e0ab4..926c7ce58b293 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -56,7 +56,6 @@
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheDeploymentManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index a5f647aa3d2b9..12b02f0fa9386 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -372,7 +372,7 @@ public void onEntryExpired(GridCacheEntryEx e, KeyCacheObject key, CacheObject o
         if (F.isEmpty(lsnrCol))
             return;
 
-        boolean primary = cctx.affinity().primary(cctx.localNode(), e.partition(), AffinityTopologyVersion.NONE);
+        boolean primary = cctx.affinity().primaryByPartition(cctx.localNode(), e.partition(), AffinityTopologyVersion.NONE);
 
         if (cctx.isReplicated() || primary) {
             boolean recordIgniteEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 18c301124405f..b07a1175c436d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -1288,7 +1288,7 @@ protected void batchStoreCommit(Iterable<IgniteTxEntry> writeEntries) throws Ign
                         if (!skip && skipNonPrimary) {
                             skip = e.cached().isNear() ||
                                 e.cached().detached() ||
-                                !e.context().affinity().primary(e.cached().partition(), topologyVersion()).isLocal();
+                                !e.context().affinity().primaryByPartition(e.cached().partition(), topologyVersion()).isLocal();
                         }
 
                         if (!skip && !local() && // Update local store at backups only if needed.
@@ -1707,7 +1707,7 @@ protected boolean isNearLocallyMapped(IgniteTxEntry e, boolean primaryOnly) {
 
         int part = cached != null ? cached.partition() : cacheCtx.affinity().partition(e.key());
 
-        List<ClusterNode> affNodes = cacheCtx.affinity().nodes(part, topologyVersion());
+        List<ClusterNode> affNodes = cacheCtx.affinity().nodesByPartition(part, topologyVersion());
 
         e.locallyMapped(F.contains(affNodes, cctx.localNode()));
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 1a9b082b3793f..7ac439807a61f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1112,7 +1112,7 @@ assert isWriteToStoreFromDhtValid(stores) :
      * @return {@code True} if local node is current primary for given entry.
      */
     private boolean primaryLocal(GridCacheEntryEx entry) {
-        return entry.context().affinity().primary(cctx.localNode(), entry.partition(), AffinityTopologyVersion.NONE);
+        return entry.context().affinity().primaryByPartition(cctx.localNode(), entry.partition(), AffinityTopologyVersion.NONE);
     }
 
     /**
@@ -1413,7 +1413,7 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                     finally {
                         if (entry != null && readCommitted()) {
                             if (cacheCtx.isNear()) {
-                                if (cacheCtx.affinity().belongs(cacheCtx.localNode(), entry.partition(), topVer)) {
+                                if (cacheCtx.affinity().partitionBelongs(cacheCtx.localNode(), entry.partition(), topVer)) {
                                     if (entry.markObsolete(xidVer))
                                         cacheCtx.cache().removeEntry(entry);
                                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
index 70d938e144ead..67d00ea782923 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
@@ -401,7 +401,7 @@ private void mapTxKeys(@Nullable Set<IgniteTxKey> txKeys, Map<IgniteTxKey, TxLoc
         private UUID primary(IgniteTxKey txKey) {
             GridCacheContext ctx = cctx.cacheContext(txKey.cacheId());
 
-            ClusterNode node = ctx.affinity().primary(txKey.key(), topVer);
+            ClusterNode node = ctx.affinity().primaryByKey(txKey.key(), topVer);
 
             assert node != null : topVer;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index 515314eb08728..a8ae2eaaee582 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -1945,7 +1945,7 @@ else if (ttl == CU.TTL_NOT_CHANGED)
                         expiryTime = CU.toExpireTime(ttl);
                     }
 
-                    boolean primary = cctx.affinity().primary(cctx.localNode(), entry.key(), topVer);
+                    boolean primary = cctx.affinity().primaryByKey(cctx.localNode(), entry.key(), topVer);
 
                     entry.initialValue(e.getValue(),
                         ver,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
index 70232afe717fa..d5f277ebf3f16 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
@@ -496,7 +496,7 @@ private Collection<ClusterNode> dataNodes(AffinityTopologyVersion topVer) throws
         Collection<ClusterNode> nodes;
 
         if (collocated) {
-            List<ClusterNode> nodes0 = ctx.affinity().nodes(hdrPart, topVer);
+            List<ClusterNode> nodes0 = ctx.affinity().nodesByPartition(hdrPart, topVer);
 
             nodes = !nodes0.isEmpty() ?
                 Collections.singleton(nodes0.contains(ctx.localNode()) ? ctx.localNode() : F.first(nodes0)) : nodes0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
index e8b2cc7f0f4b8..bc6c1827a0774 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
@@ -91,7 +91,7 @@ public IgniteUuid setId() {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public boolean apply(K k, V v) {
-        return !filter || ctx.affinity().primary(ctx.localNode(), k, ctx.affinity().affinityTopologyVersion());
+        return !filter || ctx.affinity().primaryByKey(ctx.localNode(), k, ctx.affinity().affinityTopologyVersion());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index 7889a5579c7bb..992e02a6fc570 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -1568,7 +1568,7 @@ public PartitionsReservation(int[] cacheIds, int partId,
                         }
                     }
                     finally {
-                        if (checkPartMapping && !cctx.affinity().primary(partId, topVer).id().equals(ctx.localNodeId()))
+                        if (checkPartMapping && !cctx.affinity().primaryByPartition(partId, topVer).id().equals(ctx.localNodeId()))
                             throw new IgniteException("Failed partition reservation. " +
                                 "Partition is not primary on the node. [partition=" + partId + ", cacheName=" + cctx.name() +
                                 ", nodeId=" + ctx.localNodeId() + ", topology=" + topVer + ']');
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
index 92e2b9b2ba21d..b0337d62967e0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
@@ -214,12 +214,12 @@ public CheckCallable(Object key, AffinityTopologyVersion topVer) {
 
                 ClusterNode loc = ignite.cluster().localNode();
 
-                if (loc.equals(aff.primary(key, topVer)))
+                if (loc.equals(aff.primaryByKey(key, topVer)))
                     return true;
 
                 AffinityTopologyVersion topVer0 = new AffinityTopologyVersion(topVer.topologyVersion() + 1, 0);
 
-                assertEquals(loc, aff.primary(key, topVer0));
+                assertEquals(loc, aff.primaryByKey(key, topVer0));
             }
 
             return null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
index 1cfb330da8cd6..53b4900410033 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
@@ -5837,7 +5837,7 @@ public CheckEntriesTask(Collection<String> keys) {
             int size = 0;
 
             for (String key : keys) {
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx())) {
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx())) {
                     GridCacheEntryEx e =
                         ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key);
 
@@ -5873,7 +5873,7 @@ private static class CheckCacheSizeTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : map.keySet())
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(ctx.name()).localSize(ALL));
@@ -6116,7 +6116,7 @@ public CheckKeySizeTask(Collection<String> keys) {
             int size = 0;
 
             for (String key : keys)
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(null).localSize(ALL));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
index 6b0e1932e1f2c..d4449f9ee9228 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
@@ -5548,7 +5548,7 @@ private static class CheckEntriesTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : keys) {
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx())) {
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx())) {
                     GridCacheEntryEx e =
                         ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key);
 
@@ -5589,7 +5589,7 @@ private static class CheckCacheSizeTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : map.keySet())
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(ctx.name()).localSize(ALL));
@@ -5850,7 +5850,7 @@ public CheckKeySizeTask(Collection<String> keys, String s) {
             int size = 0;
 
             for (String key : keys)
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(cacheName).localSize(ALL));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
index 6c577c63ab693..fac24ccd00045 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
@@ -1009,9 +1009,9 @@ private T2<Integer, Integer> swapKeysCount(int nodeIdx, int part) throws IgniteC
         //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primary(ctx.localNode(), part, topVer))
+        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             primaryCnt = cnt;
-        else if (affinity.backup(ctx.localNode(), part, topVer))
+        else if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             backupCnt = cnt;
         return new T2<>(primaryCnt, backupCnt);
     }
@@ -1081,9 +1081,9 @@ private T2<Integer, Integer> offheapKeysCount(int nodeIdx, int part) throws Igni
         //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primary(ctx.localNode(), part, topVer))
+        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             primaryCnt = cnt;
-        else if (affinity.backup(ctx.localNode(), part, topVer))
+        else if (affinity.backupByPartition(ctx.localNode(), part, topVer))
             backupCnt = cnt;
         return new T2<>(primaryCnt, backupCnt);
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
index 45b4b9f299aa3..aeca2fb2279ff 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
@@ -365,7 +365,7 @@ private int primaryQueueNode(IgniteQueue queue) {
 
         for (int i = 0; i < gridCount(); i++) {
             for (GridCacheEntryEx e : ((IgniteKernal)grid(i)).context().cache().internalCache(cctx.name()).allEntries()) {
-                if (aff.primary(grid(i).localNode(), e.key(), AffinityTopologyVersion.NONE)
+                if (aff.primaryByKey(grid(i).localNode(), e.key(), AffinityTopologyVersion.NONE)
                     && e.key().value(cctx.cacheObjectContext(), false) instanceof GridCacheQueueHeaderKey)
                     return i;
             }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
index b4ef11a82c0c1..8709d05664d94 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
@@ -853,13 +853,13 @@ public void testPessimisticTx2() throws Exception {
 
         GridCacheAffinityManager aff = ignite0.context().cache().internalCache(null).context().affinity();
 
-        List<ClusterNode> nodes1 = aff.nodes(key1, topVer1);
-        List<ClusterNode> nodes2 = aff.nodes(key1, topVer2);
+        List<ClusterNode> nodes1 = aff.nodesByKey(key1, topVer1);
+        List<ClusterNode> nodes2 = aff.nodesByKey(key1, topVer2);
 
         assertEquals(nodes1, nodes2);
 
-        nodes1 = aff.nodes(key2, topVer1);
-        nodes2 = aff.nodes(key2, topVer2);
+        nodes1 = aff.nodesByKey(key2, topVer1);
+        nodes2 = aff.nodesByKey(key2, topVer2);
 
         assertFalse(nodes1.get(0).equals(nodes2.get(0)));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
index aa240aae439a9..b909de90fd42a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
@@ -286,7 +286,7 @@ private void doTestDeadlock(
                     key = keys.get(1);
 
                     ClusterNode primaryNode =
-                        ((IgniteCacheProxy)cache).context().affinity().primary(key, NONE);
+                        ((IgniteCacheProxy)cache).context().affinity().primaryByKey(key, NONE);
 
                     List<Integer> primaryKeys =
                         primaryKeys(grid(primaryNode).cache(CACHE_NAME), 5, key + (100 * threadNum));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
index 83eb908d21978..ced8b61166876 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
@@ -293,7 +293,7 @@ private void doTestDeadlock(
                     key = keys.get(1);
 
                     ClusterNode primaryNode =
-                        ((IgniteCacheProxy)cache).context().affinity().primary(key, NONE);
+                        ((IgniteCacheProxy)cache).context().affinity().primaryByKey(key, NONE);
 
                     List<Integer> primaryKeys =
                         primaryKeys(grid(primaryNode).cache(CACHE_NAME), 5, key + (100 * threadNum));
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 8b2993f263cf7..155edeeeeb229 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -2061,7 +2061,7 @@ private void createSqlFunctions(String schema, Class<?>[] clss) throws IgniteChe
 
                 return new IgniteBiPredicate<K, V>() {
                     @Override public boolean apply(K k, V v) {
-                        return aff.primary(locNode, k, topVer0);
+                        return aff.primaryByKey(locNode, k, topVer0);
                     }
                 };
             }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 22b94c7ac796b..3700774260ded 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -612,7 +612,7 @@ private ClusterNode rangeNode(GridCacheContext<?,?> cctx, GridH2QueryContext qct
             node = cctx.discovery().node(nodeId);
         }
         else // Get primary node for current topology version.
-            node = cctx.affinity().primary(affKeyObj, qctx.topologyVersion());
+            node = cctx.affinity().primaryByKey(affKeyObj, qctx.topologyVersion());
 
         if (node == null) // Node was not found, probably topology changed and we need to retry the whole query.
             throw new GridH2RetryException("Failed to find node.");

From 4b20d03c148c83bf3665d3296ecbf8f768a43e0c Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Wed, 30 Aug 2017 18:12:17 +0300
Subject: [PATCH 344/516] gg-12723 Optimistic tx recovery fixed

---
 .../cache/distributed/dht/GridDhtTxLocal.java |   5 +-
 ...iteTxRecoveryAfterStoreCommitSelfTest.java | 363 ++++++++++++++++++
 .../IgniteCacheTxRecoverySelfTestSuite.java   |  10 +-
 3 files changed, 369 insertions(+), 9 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteTxRecoveryAfterStoreCommitSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
index b659abb8ef679..9220c9ebd7e91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
@@ -437,9 +437,12 @@ public IgniteInternalFuture<GridNearTxPrepareResponse> prepareAsync(
 
         if (state() != PREPARING) {
             if (!state(PREPARING)) {
-                if (state() == PREPARED && isSystemInvalidate())
+                if (state() == PREPARED && isSystemInvalidate()) {
                     fut.complete();
 
+                    return fut;
+                }
+
                 if (setRollbackOnly()) {
                     if (timeout == -1)
                         fut.onError(new IgniteTxTimeoutCheckedException("Transaction timed out and was rolled back: " +
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteTxRecoveryAfterStoreCommitSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteTxRecoveryAfterStoreCommitSelfTest.java
new file mode 100644
index 0000000000000..fa37f23ed887f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteTxRecoveryAfterStoreCommitSelfTest.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
+import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteFutureTimeoutException;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.apache.ignite.util.TestTcpCommunicationSpi;
+import org.jsr166.ConcurrentHashMap8;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+
+/**
+ * Tests that transaction is invalidated in case of {@link IgniteTxHeuristicCheckedException}.
+ */
+public class IgniteTxRecoveryAfterStoreCommitSelfTest extends GridCacheAbstractSelfTest {
+    /** Cache name. */
+    private static final String CACHE_NAME = "cache";
+
+    /** Store map. */
+    public static final Map<Object, Object> storeMap = new ConcurrentHashMap8<>();
+
+    /** */
+    private static volatile CountDownLatch storeCommitLatch;
+
+    /** */
+    private static volatile CountDownLatch nodeFailLatch;
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 5;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        startGridsMultiThreaded(gridCount());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @return Index of node starting transaction.
+     */
+    protected int originatingNode() {
+        return 0;
+    }
+
+    @Override protected long getTestTimeout() {
+        return 300_000;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        TcpCommunicationSpi comm = new TestTcpCommunicationSpi();
+
+        comm.setSharedMemoryPort(-1);
+
+        TestTcpDiscoverySpi discoSpi = new TestTcpDiscoverySpi();
+
+        discoSpi.setIpFinder(GridCacheAbstractSelfTest.ipFinder);
+
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName)
+                .setFailureDetectionTimeout(5_000)
+                .setDiscoverySpi(discoSpi)
+                .setCommunicationSpi(comm);
+
+        if (igniteInstanceName.endsWith("0"))
+            cfg.setUserAttributes(Collections.singletonMap("ORIGINATOR", true));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
+        CacheConfiguration cfg = super.cacheConfiguration(igniteInstanceName);
+
+        cfg.setName(CACHE_NAME);
+
+        cfg.setCacheStoreFactory(new TestStoreFactory());
+
+        cfg.setReadThrough(true);
+        cfg.setWriteThrough(true);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testManyKeysCommit() throws Exception {
+        Collection<Integer> keys = new ArrayList<>(200);
+
+        for (int i = 0; i < 20; i++)
+            keys.add(i);
+
+        testTxOriginatingNodeFails(keys);
+    }
+
+    /**
+     * @param keys Keys to update.
+     * @throws Exception If failed.
+     */
+    protected void testTxOriginatingNodeFails(final Collection<Integer> keys) throws Exception {
+        assertFalse(keys.isEmpty());
+
+        final Collection<IgniteKernal> grids = new ArrayList<>();
+
+        ClusterNode txNode = grid(originatingNode()).localNode();
+
+        for (int i = 1; i < gridCount(); i++)
+            grids.add((IgniteKernal)grid(i));
+
+        final Map<Integer, String> expectedStoreState = new HashMap<>();
+
+        final String initVal = "initialValue";
+
+        for (Integer key : keys) {
+            grid(originatingNode()).cache(CACHE_NAME).put(key, initVal);
+
+            expectedStoreState.put(key, String.valueOf(key));
+        }
+
+        Map<Integer, Collection<ClusterNode>> nodeMap = new HashMap<>();
+
+        info("Node being checked: " + grid(1).localNode().id());
+
+        for (Integer key : keys) {
+            Collection<ClusterNode> nodes = new ArrayList<>();
+
+            nodes.addAll(grid(1).affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key));
+
+            nodes.remove(txNode);
+
+            nodeMap.put(key, nodes);
+        }
+
+        info("Starting tx [values=" + expectedStoreState + ", topVer=" +
+                grid(1).context().discovery().topologyVersion() + ']');
+
+        final IgniteEx originatingNodeGrid = grid(originatingNode());
+
+        storeCommitLatch = new CountDownLatch(1);
+
+        nodeFailLatch = new CountDownLatch(1);
+
+        GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                IgniteCache<Integer, String> cache = originatingNodeGrid.cache(CACHE_NAME);
+
+                assertNotNull(cache);
+
+                Transaction tx = originatingNodeGrid.transactions().txStart(OPTIMISTIC, TransactionIsolation.SERIALIZABLE);
+
+                try {
+                    cache.putAll(expectedStoreState);
+
+                    info("Before commit");
+
+                    tx.commit();
+                }
+                catch (IgniteFutureTimeoutException ignored) {
+                    info("Failed to wait for commit future completion");
+                }
+
+                return null;
+            }
+        });
+
+        nodeFailLatch.await();
+
+        for (Integer key : expectedStoreState.keySet())
+            assertEquals(expectedStoreState.get(key), storeMap.get(key));
+
+        info(">>> Stopping originating node " + txNode);
+
+        ((TestTcpDiscoverySpi)grid(originatingNode()).context().config().getDiscoverySpi()).simulateNodeFailure();
+        ((TestTcpCommunicationSpi)grid(originatingNode()).context().config().getCommunicationSpi()).simulateNodeFailure();
+
+        storeCommitLatch.countDown();
+
+        G.stop(grid(originatingNode()).name(), true);
+
+        info(">>> Stopped originating node: " + txNode.id());
+
+        boolean txFinished = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                for (IgniteKernal g : grids) {
+                    GridCacheAdapter<?, ?> cache = g.internalCache(CACHE_NAME);
+
+                    IgniteTxManager txMgr = cache.isNear() ?
+                            ((GridNearCacheAdapter)cache).dht().context().tm() :
+                            cache.context().tm();
+
+                    int txNum = txMgr.idMapSize();
+
+                    if (txNum != 0)
+                        return false;
+                }
+
+                return true;
+            }
+        }, 300_000);
+
+        assertTrue(txFinished);
+
+        info("Transactions finished.");
+
+        for (Map.Entry<Integer, Collection<ClusterNode>> e : nodeMap.entrySet()) {
+            final Integer key = e.getKey();
+
+            final String val = expectedStoreState.get(key);
+
+            assertFalse(e.getValue().isEmpty());
+
+            for (ClusterNode node : e.getValue()) {
+                final UUID checkNodeId = node.id();
+
+                compute(G.ignite(checkNodeId).cluster().forNode(node)).call(new IgniteCallable<Void>() {
+                    /** */
+                    @IgniteInstanceResource
+                    private Ignite ignite;
+
+                    @Override public Void call() throws Exception {
+                        IgniteCache<Integer, String> cache = ignite.cache(CACHE_NAME);
+
+                        assertNotNull(cache);
+
+                        assertEquals("Failed to check entry value on node: " + checkNodeId,
+                                val, cache.get(key));
+
+                        return null;
+                    }
+                });
+            }
+        }
+
+        for (Map.Entry<Integer, String> e : expectedStoreState.entrySet()) {
+            for (Ignite g : G.allGrids())
+                assertEquals(e.getValue(), g.cache(CACHE_NAME).get(e.getKey()));
+        }
+    }
+
+    /**
+     *
+     */
+    public static class TestStoreFactory implements Factory<CacheStore> {
+        @IgniteInstanceResource
+        Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public CacheStore create() {
+            return new TestStore(ignite.cluster().localNode().attribute("ORIGINATOR") != null);
+        }
+    }
+
+    /**
+     *
+     */
+    public static class TestStore extends CacheStoreAdapter<Object, Object> {
+        /** */
+        private boolean originatorNodeFlag;
+
+        /** */
+        public TestStore(boolean originatorNodeFlag) {
+
+            this.originatorNodeFlag = originatorNodeFlag;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object load(Object key) {
+            return storeMap.get(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Object, ? extends Object> entry) {
+            storeMap.put(entry.getKey(), entry.getValue());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) {
+            storeMap.remove(key);
+        }
+
+        @Override public void sessionEnd(boolean commit) {
+            if (!originatorNodeFlag)
+                return;
+
+            if (storeCommitLatch != null) {
+                try {
+                    nodeFailLatch.countDown();
+
+                    storeCommitLatch.await();
+
+                    throw new IgniteException();
+                }
+                catch (InterruptedException e) {
+                    throw new IgniteException(e);
+                }
+            }
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
index c7c8db67fe31e..e901cf5489220 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTxRecoverySelfTestSuite.java
@@ -18,14 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedTxPessimisticOriginatingNodeFailureSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledTxOriginatingNodeFailureSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedTxOriginatingNodeFailureSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheCommitDelayTxRecoveryTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedNearDisabledPrimaryNodeFailureRecoveryTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedPrimaryNodeFailureRecoveryTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedTwoBackupsPrimaryNodeFailureRecoveryTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.TxRecoveryStoreEnabledTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.*;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearTxPessimisticOriginatingNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxOriginatingNodeFailureSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxPessimisticOriginatingNodeFailureSelfTest;
@@ -56,6 +49,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheReplicatedTxPessimisticOriginatingNodeFailureSelfTest.class);
 
         suite.addTestSuite(TxRecoveryStoreEnabledTest.class);
+        suite.addTestSuite(IgniteTxRecoveryAfterStoreCommitSelfTest.class);
 
         return suite;
     }

From e7b1157d008d7416c60a511274f7f85aa919433e Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Mon, 21 Aug 2017 17:54:12 +0300
Subject: [PATCH 345/516] GG-12588 Backport IGNITE-5943 Communication. Server
 node may reject client connection during massive clients join.

---
 .../tcp/TcpCommunicationSpi.java              |  61 ++++++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  24 ++-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |  10 +
 .../tcp/IgniteClientConnectTest.java          | 172 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   3 +
 5 files changed, 263 insertions(+), 7 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index ed6789caa9843..de1f76d9f1b61 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -65,6 +65,7 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.GridConcurrentFactory;
 import org.apache.ignite.internal.util.GridSpinReadWriteLock;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -129,6 +130,8 @@
 import org.apache.ignite.spi.IgniteSpiTimeoutObject;
 import org.apache.ignite.spi.communication.CommunicationListener;
 import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentLinkedDeque8;
@@ -137,6 +140,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
+import static org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.RecoveryLastReceivedMessage.NEED_WAIT;
 
 /**
  * <tt>TcpCommunicationSpi</tt> is default communication SPI which uses
@@ -301,6 +305,13 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
     /** Connection index meta for session. */
     private static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
 
+    /**
+     * Version when client is ready to wait to connect to server (could be needed when client tries to open connection
+     * before it starts being visible for server)
+     */
+    private static final IgniteProductVersion VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT =
+        IgniteProductVersion.fromString("1.9.7");
+
     /** Message tracker meta for session. */
     private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
 
@@ -435,7 +446,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
              * @param ses Session.
              * @param msg Message.
              */
-            private void onFirstMessage(GridNioSession ses, Message msg) {
+            private void onFirstMessage(final GridNioSession ses, Message msg) {
                 UUID sndId;
 
                 ConnectionKey connKey;
@@ -459,10 +470,35 @@ private void onFirstMessage(GridNioSession ses, Message msg) {
                 final ClusterNode rmtNode = getSpiContext().node(sndId);
 
                 if (rmtNode == null) {
-                    U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId +
-                        ", ses=" + ses + ']');
+                    DiscoverySpi discoverySpi = ignite.configuration().getDiscoverySpi();
+
+                    assert discoverySpi instanceof TcpDiscoverySpi;
+
+                    TcpDiscoverySpi tcpDiscoverySpi = (TcpDiscoverySpi) discoverySpi;
+
+                    ClusterNode node0 = tcpDiscoverySpi.getNode0(sndId);
+
+                    boolean unknownNode = true;
 
-                    ses.close();
+                    if (node0 != null) {
+                        assert node0.isClient() : node0;
+
+                        if (node0.version().compareTo(VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT) >= 0)
+                            unknownNode = false;
+                    }
+
+                    if (unknownNode) {
+                        U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId + ", ses=" + ses + ']');
+
+                        ses.close();
+                    }
+                    else {
+                        ses.send(new RecoveryLastReceivedMessage(NEED_WAIT)).listen(new CI1<IgniteInternalFuture<?>>() {
+                            @Override public void apply(IgniteInternalFuture<?> fut) {
+                                ses.close();
+                            }
+                        });
+                    }
 
                     return;
                 }
@@ -2858,6 +2894,8 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
             IgniteSpiOperationTimeoutHelper timeoutHelper = new IgniteSpiOperationTimeoutHelper(this);
 
+            int lastWaitingTimeout = 1;
+
             while (!conn) { // Reconnection on handshake timeout.
                 try {
                     SocketChannel ch = SocketChannel.open();
@@ -2920,6 +2958,18 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
 
                         if (rcvCnt == -1)
                             return null;
+                        else if (rcvCnt == NEED_WAIT) {
+                            recoveryDesc.release();
+
+                            U.closeQuiet(ch);
+
+                            if (lastWaitingTimeout < 60000)
+                                lastWaitingTimeout *= 2;
+
+                            U.sleep(lastWaitingTimeout);
+
+                            continue;
+                        }
                     }
                     finally {
                         if (recoveryDesc != null && rcvCnt == -1)
@@ -4294,6 +4344,9 @@ public static class RecoveryLastReceivedMessage implements Message {
         /** */
         private static final long serialVersionUID = 0L;
 
+        /** Need wait. */
+        static final long NEED_WAIT = -3;
+
         /** */
         private long rcvCnt;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 84343e46a9521..e1dff1c0ad1d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1803,9 +1803,27 @@ private static void removeMetrics(TcpDiscoveryHeartbeatMessage msg, UUID nodeId)
     }
 
     /**
-     * Thread that cleans IP finder and keeps it in the correct state, unregistering addresses of the nodes that has
-     * left the topology. <p> This thread should run only on coordinator node and will clean IP finder if and only if
-     * {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()} is {@code true}.
+     * Trying get node in any state (visible or not)
+     * @param nodeId Node id.
+     */
+    ClusterNode getNode0(UUID nodeId) {
+        assert nodeId != null;
+
+        UUID locNodeId0 = getLocalNodeId();
+
+        if (locNodeId0 != null && locNodeId0.equals(nodeId))
+            // Return local node directly.
+            return locNode;
+
+        return ring.node(nodeId);
+    }
+
+    /**
+     * Thread that cleans IP finder and keeps it in the correct state, unregistering
+     * addresses of the nodes that has left the topology.
+     * <p>
+     * This thread should run only on coordinator node and will clean IP finder
+     * if and only if {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()} is {@code true}.
      */
     private class IpFinderCleaner extends IgniteSpiThread {
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 8c7ef99afcbff..ac423363184ea 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -437,6 +437,16 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, T
         return impl.getNode(nodeId);
     }
 
+    /**
+     * @param id Id.
+     */
+    public ClusterNode getNode0(UUID id) {
+        if (impl instanceof ServerImpl)
+            return ((ServerImpl)impl).getNode0(id);
+
+        return getNode(id);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean pingNode(UUID nodeId) {
         return impl.pingNode(nodeId);
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java
new file mode 100644
index 0000000000000..b1c4dbf25434d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientConnectTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+
+/**
+ * We emulate that client receive message about joining to topology earlier than some server nodes in topology.
+ * And make this client connect to such servers.
+ * To emulate this we connect client to second node in topology and pause sending message about joining finishing to
+ * third node.
+ */
+public class IgniteClientConnectTest extends GridCommonAbstractTest {
+
+    /** Custom cache name. */
+    private static final String DEFAULT_CACHE_NAME = "default-cache";
+
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** Latch to stop message sending. */
+    private final CountDownLatch latch = new CountDownLatch(1);
+
+    /** Start client flag. */
+    private final AtomicBoolean clientJustStarted = new AtomicBoolean(false);
+
+    /** Instance name. */
+    private String igniteInstanceName;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        this.igniteInstanceName = igniteInstanceName;
+
+        TestTcpDiscoverySpi disco = new TestTcpDiscoverySpi();
+
+        if (igniteInstanceName.equals("client")) {
+            TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
+
+            ipFinder.registerAddresses(Collections.singleton(new InetSocketAddress(InetAddress.getLoopbackAddress(), 47501)));
+
+            disco.setIpFinder(ipFinder);
+        }
+        else
+            disco.setIpFinder(ipFinder);
+
+        disco.setJoinTimeout(2 * 60_000);
+        disco.setSocketTimeout(1000);
+        disco.setNetworkTimeout(2000);
+
+        cfg.setDiscoverySpi(disco);
+
+        CacheConfiguration cacheConfiguration = new CacheConfiguration()
+                .setName(DEFAULT_CACHE_NAME)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setAffinity(new RendezvousAffinityFunction(false, 8))
+                .setBackups(0);
+
+        cfg.setCacheConfiguration(cacheConfiguration);
+
+        return cfg;
+    }
+
+    /**
+     *
+     * @throws Exception If failed.
+     */
+    public void testClientConnectToBigTopology() throws Exception {
+        Ignite ignite = startGrids(3);
+
+        IgniteCache<Object, Object> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        Set<Integer> keys = new HashSet<>();
+
+        for (int i = 0; i < 80; i++) {
+            cache.put(i, i);
+
+            keys.add(i);
+        }
+
+        TcpDiscoveryImpl discovery = ((TestTcpDiscoverySpi) ignite.configuration().getDiscoverySpi()).discovery();
+
+        assertTrue(discovery instanceof ServerImpl);
+
+        IgniteConfiguration clientCfg = getConfiguration("client");
+
+        clientCfg.setClientMode(true);
+
+        clientJustStarted.set(true);
+
+        Ignite client = startGrid(igniteInstanceName, clientCfg);
+
+        latch.countDown();
+
+        System.err.println("GET ALL");
+        client.cache(DEFAULT_CACHE_NAME).getAll(keys);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     *
+     */
+    class TestTcpDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg, long timeout) throws IOException,
+                IgniteCheckedException {
+            if (msg instanceof TcpDiscoveryNodeAddFinishedMessage) {
+                if (msg.senderNodeId() != null && clientJustStarted.get())
+                    try {
+                        latch.await();
+
+                        Thread.sleep(3000);
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    }
+
+                super.writeToSocket(sock, out, msg, timeout);
+            }
+            else
+                super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /**
+         *
+         */
+        TcpDiscoveryImpl discovery() {
+            return impl;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index fa470e911790a..c506ca7e95bf2 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.spi.GridTcpSpiForwardingSelfTest;
 import org.apache.ignite.spi.discovery.AuthenticationRestartTest;
+import org.apache.ignite.spi.discovery.tcp.IgniteClientConnectTest;
 import org.apache.ignite.spi.discovery.tcp.IgniteClientReconnectMassiveShutdownTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
@@ -91,6 +92,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
         suite.addTest(new TestSuite(AuthenticationRestartTest.class));
 
+        //Client connect
+        suite.addTest(new TestSuite(IgniteClientConnectTest.class));
         suite.addTest(new TestSuite(IgniteClientReconnectMassiveShutdownTest.class));
 
         // SSL.

From dbc340a37664f8e5313ab085af34f59ce963032b Mon Sep 17 00:00:00 2001
From: Konstantin Boudnik <konstantin_boudnik@epam.com>
Date: Mon, 5 Jun 2017 19:47:02 -0700
Subject: [PATCH 346/516] IGNITE-5413. Ignite shouldn't expose nor send
 (clear-text) env variables to a 3rd endpoint

---
 .../ignite/internal/processors/cluster/ClusterProcessor.java | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
index 6500cf3e8faee..1d02749d009cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
@@ -58,7 +58,7 @@ public class ClusterProcessor extends GridProcessorAdapter {
     private IgniteClusterImpl cluster;
 
     /** */
-    private final AtomicBoolean notifyEnabled = new AtomicBoolean();
+    private final AtomicBoolean notifyEnabled = new AtomicBoolean(false);
 
     /** */
     @GridToStringExclude
@@ -74,9 +74,6 @@ public class ClusterProcessor extends GridProcessorAdapter {
     public ClusterProcessor(GridKernalContext ctx) {
         super(ctx);
 
-        notifyEnabled.set(IgniteSystemProperties.getBoolean(IGNITE_UPDATE_NOTIFIER,
-            Boolean.parseBoolean(IgniteProperties.get("ignite.update.notifier.enabled.by.default"))));
-
         cluster = new IgniteClusterImpl(ctx);
     }
 

From 825409fb17e414cdb376a96074b62262eead859a Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Thu, 13 Jul 2017 18:34:01 +0300
Subject: [PATCH 347/516] Functionality of GridVersionSelfTest is debated now

---
 .../java/org/apache/ignite/internal/GridVersionSelfTest.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridVersionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridVersionSelfTest.java
index 4751a0c4fb49b..13af907489f4f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridVersionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridVersionSelfTest.java
@@ -32,6 +32,8 @@ public class GridVersionSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testVersions() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-5413");
+
         String propVal = System.getProperty(IGNITE_UPDATE_NOTIFIER);
 
         System.setProperty(IGNITE_UPDATE_NOTIFIER, "true");

From 41d92442c16cce673523295dfe0a7ffac686003a Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Thu, 13 Jul 2017 17:32:06 +0300
Subject: [PATCH 348/516] Functionality of muted test is debated now

---
 .../internal/IgniteUpdateNotifierPerClusterSettingSelfTest.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteUpdateNotifierPerClusterSettingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteUpdateNotifierPerClusterSettingSelfTest.java
index a255f15345782..9fca528a691d5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteUpdateNotifierPerClusterSettingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteUpdateNotifierPerClusterSettingSelfTest.java
@@ -66,6 +66,8 @@ public class IgniteUpdateNotifierPerClusterSettingSelfTest extends GridCommonAbs
      * @throws Exception If failed.
      */
     public void testNotifierEnabledForCluster() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-5413");
+
         checkNotifierStatusForCluster(true);
     }
 

From d8fe81f827b4db1800276e5d76ead86e1648c224 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Wed, 16 Aug 2017 00:24:07 +0300
Subject: [PATCH 349/516] IgniteCacheNearRestartRollbackSelfTest#testRestarts
 is muted.

---
 .../distributed/IgniteCacheNearRestartRollbackSelfTest.java     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index 3f242b5726867..aea4d7782d858 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -132,6 +132,8 @@ protected CacheConfiguration<Object, Object> cacheConfiguration(String gridName)
      */
     @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
     public void testRestarts() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-12398");
+
         startGrids(4);
 
         Ignite tester = ignite(3);

From 882f4b40883b11a2cda5a86b1fa2f2af9f103d32 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Thu, 31 Aug 2017 16:50:33 +0300
Subject: [PATCH 350/516] Fixed flaky test IgniteCacheEntryListener*

---
 .../query/continuous/CacheContinuousQueryEntry.java      | 9 +--------
 .../query/continuous/CacheContinuousQueryHandler.java    | 2 --
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index 366a1e05fa46c..ffbbc2c1e05ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -199,13 +199,6 @@ void markFiltered() {
         depInfo = null;
     }
 
-    /**
-     * @param topVer Topology version.
-     */
-    void topologyVersion(AffinityTopologyVersion topVer) {
-        this.topVer = topVer;
-    }
-
     /**
      * @return Size include this event and filtered.
      */
@@ -222,7 +215,7 @@ CacheContinuousQueryEntry forBackupQueue() {
             return this;
 
         CacheContinuousQueryEntry e =
-            new CacheContinuousQueryEntry(cacheId, null, null, null, null, keepBinary, part, updateCntr, null);
+            new CacheContinuousQueryEntry(cacheId, null, null, null, null, keepBinary, part, updateCntr, topVer);
 
         e.flags = flags;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 926c7ce58b293..0cdbec5013b15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -465,8 +465,6 @@ public void keepBinary(boolean keepBinary) {
                         for (CacheContinuousQueryEntry e : backupQueue0) {
                             if (!e.isFiltered())
                                 prepareEntry(cctx, nodeId, e);
-
-                            e.topologyVersion(topVer);
                         }
 
                         ctx.continuous().addBackupNotification(nodeId, routineId, backupQueue0, topic);

From fcdf7a874147c1c9e4241a6663e320c82588d4bd Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 1 Sep 2017 14:52:32 +0300
Subject: [PATCH 351/516] ignite-6053: fixed clear() on local cache

---
 .../processors/cache/GridCacheAdapter.java    | 42 +++++++--
 .../local/GridCacheLocalFullApiSelfTest.java  | 92 ++++++++++++++++++-
 2 files changed, 127 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 9b1e0cc7721c1..2d2b4f2ce12b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1196,8 +1196,17 @@ public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, bool
      * @throws IgniteCheckedException In case of error.
      */
     private void clear(@Nullable Set<? extends K> keys) throws IgniteCheckedException {
-        executeClearTask(keys, false).get();
-        executeClearTask(keys, true).get();
+        if (isLocal()) {
+            if (keys == null)
+                clearLocally(true, false, false);
+            else
+                clearLocallyAll(keys, true, false, false);
+        }
+        else {
+            executeClearTask(keys, false).get();
+
+            executeClearTask(keys, true).get();
+        }
     }
 
     /**
@@ -1205,13 +1214,34 @@ private void clear(@Nullable Set<? extends K> keys) throws IgniteCheckedExceptio
      * @return Future.
      */
     private IgniteInternalFuture<?> clearAsync(@Nullable final Set<? extends K> keys) {
-        return executeClearTask(keys, false).chain(new CX1<IgniteInternalFuture<?>, Object>() {
-            @Override public Object applyx(IgniteInternalFuture<?> fut) throws IgniteCheckedException {
-                executeClearTask(keys, true).get();
+        if (isLocal())
+            return clearLocallyAsync(keys);
+        else {
+            return executeClearTask(keys, false).chain(new CX1<IgniteInternalFuture<?>, Object>() {
+                @Override public Object applyx(IgniteInternalFuture<?> fut) throws IgniteCheckedException {
+                    executeClearTask(keys, true).get();
+
+                    return null;
+                }
+            });
+        }
+    }
+
+    /**
+     * @param keys Keys to clear.
+     * @return Clear future.
+     */
+    private IgniteInternalFuture<?> clearLocallyAsync(@Nullable Set<? extends K> keys) {
+        return ctx.closures().callLocalSafe(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                if (keys == null)
+                    clearLocally(true, false, false);
+                else
+                    clearLocallyAll(keys, true, false, false);
 
                 return null;
             }
-        });
+        }, false);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalFullApiSelfTest.java
index f499c26d547c4..8a844bcaa6daa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalFullApiSelfTest.java
@@ -17,7 +17,9 @@
 
 package org.apache.ignite.internal.processors.cache.local;
 
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Map;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
@@ -89,4 +91,92 @@ public void testMapKeysToNodes() throws Exception {
         for (String key : keys)
             assert "key1".equals(key) || "key2".equals(key);
     }
-}
\ No newline at end of file
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLocalClearAsync() throws Exception {
+        localCacheClear(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLocalClear() throws Exception {
+        localCacheClear(false);
+    }
+
+    /**
+     * @param async If {@code true} uses async method.
+     * @throws Exception If failed.
+     */
+    private void localCacheClear(boolean async) throws Exception {
+        // In addition to the existing tests, it confirms the data is cleared only on one node,
+        // not on all nodes that have local caches with same names.
+        try {
+            startGrid(1);
+
+            IgniteCache<String, Integer> cache = jcache();
+
+            IgniteCache<String, Integer> asyncCache = cache.withAsync();
+
+            for (int i = 0; i < 5; i++) {
+                cache.put(String.valueOf(i), i);
+                jcache(1).put(String.valueOf(i), i);
+            }
+
+            if (async) {
+                asyncCache.clear("4");
+
+                asyncCache.future().get();
+            }
+            else
+                cache.clear("4");
+
+            assertNull(peek(cache, "4"));
+            assertNotNull(peek(jcache(1), "4"));
+
+            if (async) {
+                asyncCache.clearAll(new HashSet<>(Arrays.asList("2", "3")));
+
+                asyncCache.future().get();
+            }
+            else
+                cache.clearAll(new HashSet<>(Arrays.asList("2", "3")));
+
+            for (int i = 2; i < 4; i++) {
+                assertNull(peek(cache, String.valueOf(i)));
+                assertNotNull(peek(jcache(1), String.valueOf(i)));
+            }
+
+            if (async) {
+                asyncCache.clear();
+
+                asyncCache.future().get();
+            }
+            else
+                cache.clear();
+
+            for (int i = 0; i < 2; i++) {
+                assertNull(peek(cache, String.valueOf(i)));
+                assertNotNull(peek(jcache(1), String.valueOf(i)));
+            }
+
+            if (async) {
+                IgniteCache<String, Integer> asyncCache1 = jcache(1).withAsync();
+
+                asyncCache1.clear();
+
+                asyncCache1.future().get();
+            }
+            else
+                jcache(1).clear();
+
+            for (int i = 0; i < 2; i++)
+                assert jcache(i).localSize() == 0;
+        }
+        finally {
+            stopGrid(1);
+        }
+    }
+}

From eb6db758af64610745957ad5b0d165302d7972d3 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 1 Sep 2017 18:14:05 +0300
Subject: [PATCH 352/516] ignite-6053: fixed clear() on local cache ('keys'
 parameter must be final)

---
 .../ignite/internal/processors/cache/GridCacheAdapter.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 2d2b4f2ce12b4..d947aa9cb13b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1231,7 +1231,7 @@ private IgniteInternalFuture<?> clearAsync(@Nullable final Set<? extends K> keys
      * @param keys Keys to clear.
      * @return Clear future.
      */
-    private IgniteInternalFuture<?> clearLocallyAsync(@Nullable Set<? extends K> keys) {
+    private IgniteInternalFuture<?> clearLocallyAsync(@Nullable final Set<? extends K> keys) {
         return ctx.closures().callLocalSafe(new Callable<Object>() {
             @Override public Object call() throws Exception {
                 if (keys == null)

From 04143cfacf887b8bd1fd10f807b97aaa2af61ead Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 5 Sep 2017 19:11:12 +0300
Subject: [PATCH 353/516] GG-12699: Fix
 GridCacheAbstractFullApiSelfTest.testTransformResourceInjection.

Squashed commit of the following:

commit 3f20fe8dcc796406f7a7791e3ae9ddb5c26183ca
Author: Nikolay Izhikov <nizhikov.dev@gmail.com>
Date:   Wed Aug 9 13:37:11 2017 +0300

    IGNITE-5897 Fix session init/end logic. This fixes tests.

    Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>

    (cherry picked from commit 5a559df)

commit 52e89d387874a0653c58a608cc000950a76fb6b0
Author: dpavlov <dpavlov@gridgain.com>
Date:   Wed Jul 26 17:23:05 2017 +0300

    IGNITE-5806 - Fixed assertion with a side-effect - Fixes #2335.

    Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

    (cherry picked from commit 9e79c4b)

commit d3c40e418dce6ab640fe06e8c18ada4b93f1edf5
Author: Andrey V. Mashenkov <andrey.mashenkov@gmail.com>
Date:   Mon Sep 4 15:57:16 2017 +0300

    Fix javadoc.

(cherry picked from commit d4f2885)
---
 .../cache/store/GridCacheStoreManagerAdapter.java           | 3 ++-
 .../internal/processors/resource/GridResourceProcessor.java | 6 +++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 14ec92237c3d4..142e5ae28f67f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -879,7 +879,8 @@ private void sessionEnd0(@Nullable IgniteInternalTx tx, boolean threwEx) throws
                         lsnr.onSessionEnd(locSes, !threwEx);
                 }
 
-                store.sessionEnd(!threwEx);
+                if (!sesHolder.get().ended(store))
+                    store.sessionEnd(!threwEx);
             }
         }
         catch (Exception e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
index 84d07b64e50e5..02616b5e983a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
@@ -20,7 +20,6 @@
 import java.lang.annotation.Annotation;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.util.Collection;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.store.CacheStoreSession;
 import org.apache.ignite.compute.ComputeJob;
@@ -318,6 +317,11 @@ private GridResourceInjector injectorByAnnotation(GridResourceIoc.ResourceAnnota
 
     /**
      * @param obj Object to inject.
+     * @param ann Annotation enum.
+     * @param dep Grid deployment object.
+     * @param depCls Grid deployment class.
+     * @param param Resource to inject.
+     * @return {@code True} if resource was injected.
      * @throws IgniteCheckedException If failed to inject.
      */
     private boolean inject(Object obj, GridResourceIoc.ResourceAnnotation ann, @Nullable GridDeployment dep,

From 733ca74fcc094c83a3c4769b291c87d56e17c4a9 Mon Sep 17 00:00:00 2001
From: Vyacheslav Daradur <daradurvs@gmail.com>
Date: Fri, 10 Feb 2017 16:51:37 +0300
Subject: [PATCH 354/516] GG-12647: Backport IGNITE-3196 Add support for
 BigDecimals with negative scale in BinaryMarshaller (cherry picked from
 commit 5efefcb)

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../ignite/internal/binary/BinaryUtils.java   | 12 ++---
 .../internal/binary/BinaryWriterExImpl.java   | 14 +++---
 .../binary/BinaryMarshallerSelfTest.java      | 30 +++++++++++++
 modules/platforms/cpp/odbc/src/utility.cpp    | 17 ++++---
 .../Compute/ComputeApiTest.cs                 | 18 ++++----
 .../Impl/Binary/BinaryUtils.cs                | 44 +++++++++++--------
 6 files changed, 92 insertions(+), 43 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 1153d155c0400..6831ef9bb0d66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -65,8 +65,8 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 
 /**
  * Binary utils.
@@ -1193,13 +1193,15 @@ public static BigDecimal doReadDecimal(BinaryInputStream in) {
         int scale = in.readInt();
         byte[] mag = doReadByteArray(in);
 
-        BigInteger intVal = new BigInteger(mag);
+        boolean negative = mag[0] < 0;
 
-        if (scale < 0) {
-            scale &= 0x7FFFFFFF;
+        if (negative)
+            mag[0] &= 0x7F;
 
+        BigInteger intVal = new BigInteger(mag);
+
+        if (negative)
             intVal = intVal.negate();
-        }
 
         return new BigDecimal(intVal, scale);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index adaacdda40676..3289780e3fd0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -403,18 +403,20 @@ public void doWriteDecimal(@Nullable BigDecimal val) {
 
             out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL);
 
+            out.unsafeWriteInt(val.scale());
+
             BigInteger intVal = val.unscaledValue();
 
-            if (intVal.signum() == -1) {
-                intVal = intVal.negate();
+            boolean negative = intVal.signum() == -1;
 
-                out.unsafeWriteInt(val.scale() | 0x80000000);
-            }
-            else
-                out.unsafeWriteInt(val.scale());
+            if (negative)
+                intVal = intVal.negate();
 
             byte[] vals = intVal.toByteArray();
 
+            if (negative)
+                vals[0] |= -0x80;
+
             out.unsafeWriteInt(vals.length);
             out.writeByteArray(vals);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index cd8a487d6d876..31d6f31eef961 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -29,6 +29,7 @@
 import java.lang.reflect.Proxy;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.math.RoundingMode;
 import java.net.InetSocketAddress;
 import java.sql.Timestamp;
 import java.util.AbstractQueue;
@@ -178,6 +179,35 @@ public void testDecimal() throws Exception {
         assertEquals((val = new BigDecimal(new BigInteger("-79228162514264337593543950336"))), marshalUnmarshal(val));
     }
 
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -1)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -2)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE, -3)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE, -4)), marshalUnmarshal(val));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNegativeScaleRoundingModeDecimal() throws Exception {
+        BigDecimal val;
+
+        assertEquals((val = BigDecimal.ZERO.setScale(-1, RoundingMode.HALF_UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MAX_VALUE).setScale(-3, RoundingMode.HALF_DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Long.MIN_VALUE).setScale(-5, RoundingMode.HALF_EVEN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MAX_VALUE).setScale(-8, RoundingMode.UP)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Integer.MIN_VALUE).setScale(-10, RoundingMode.DOWN)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MAX_VALUE).setScale(-12, RoundingMode.CEILING)), marshalUnmarshal(val));
+        assertEquals((val = BigDecimal.valueOf(Double.MIN_VALUE).setScale(-15, RoundingMode.FLOOR)), marshalUnmarshal(val));
+    }
+
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/platforms/cpp/odbc/src/utility.cpp b/modules/platforms/cpp/odbc/src/utility.cpp
index 22191eb8b045e..2c8f9f3a3cdab 100644
--- a/modules/platforms/cpp/odbc/src/utility.cpp
+++ b/modules/platforms/cpp/odbc/src/utility.cpp
@@ -87,8 +87,14 @@ namespace ignite
 
             impl::binary::BinaryUtils::ReadInt8Array(reader.GetStream(), mag.data(), static_cast<int32_t>(mag.size()));
 
-            int32_t sign = (scale & 0x80000000) ? -1 : 1;
-            scale = scale & 0x7FFFFFFF;
+            int32_t sign = 1;
+            
+            if (mag[0] < 0)
+            {
+                mag[0] &= 0x7F;
+
+                sign = -1;
+            }
 
             common::Decimal res(mag.data(), static_cast<int32_t>(mag.size()), scale, sign);
 
@@ -101,14 +107,15 @@ namespace ignite
 
             const common::BigInteger &unscaled = decimal.GetUnscaledValue();
 
-            int32_t signFlag = unscaled.GetSign() == -1 ? 0x80000000 : 0;
-
-            writer.WriteInt32(decimal.GetScale() | signFlag);
+            writer.WriteInt32(decimal.GetScale());
 
             common::FixedSizeArray<int8_t> magnitude;
 
             unscaled.MagnitudeToBytes(magnitude);
 
+            if (unscaled.GetSign() == -1)
+                magnitude[0] |= -0x80;
+
             writer.WriteInt32(magnitude.GetSize());
 
             impl::binary::BinaryUtils::WriteInt8Array(writer.GetStream(), magnitude.GetData(), magnitude.GetSize());
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
index 71a4718130475..3a3100a6b3131 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs
@@ -817,16 +817,16 @@ public void TestEchoDecimal()
             Assert.AreEqual(val = decimal.Parse("-11,12"), _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { val, val.ToString() }));
 
             // Test echo with overflow.
-            try
-            {
-                _grid1.GetCompute().ExecuteJavaTask<object>(DecimalTask, new object[] { null, decimal.MaxValue.ToString() + 1 });
+            var ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, decimal.MaxValue.ToString() + 1}));
 
-                Assert.Fail();
-            }
-            catch (IgniteException)
-            {
-                // No-op.
-            }
+            Assert.AreEqual("Decimal magnitude overflow (must be less than 96 bits): 104", ex.Message);
+
+            // Negative scale. 1E+1 parses to "1 scale -1" on Java side.
+            ex = Assert.Throws<BinaryObjectException>(() => _grid1.GetCompute()
+                .ExecuteJavaTask<object>(DecimalTask, new object[] {null, "1E+1"}));
+
+            Assert.AreEqual("Decimal value scale overflow (must be between 0 and 28): -1", ex.Message);
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
index cc5d8a109ce86..5ce77d0ee4347 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs
@@ -893,7 +893,9 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
             // Write scale and negative flag.
             int scale = (vals[3] & 0x00FF0000) >> 16; 
 
-            stream.WriteInt(((vals[3] & 0x80000000) == 0x80000000) ? (int)((uint)scale | 0x80000000) : scale);
+            stream.WriteInt(scale);
+
+            Boolean neg = vals[3] < 0;
 
             if (idx == -1)
             {
@@ -923,13 +925,15 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             if ((part24 & 0x80) == 0x80)
                             {
                                 stream.WriteInt(len + 1);
+                                
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
 
-                                stream.WriteByte(ByteZero);
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len);
 
-                            stream.WriteByte((byte)part24);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part24 | -0x80) : part24));
                             stream.WriteByte((byte)part16);
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
@@ -940,12 +944,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 1);
-
-                            stream.WriteByte((byte)part16);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part16 | -0x80) : part16));
                             stream.WriteByte((byte)part8);
                             stream.WriteByte((byte)part0);
                         }
@@ -955,12 +961,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 1);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 2);
-
-                            stream.WriteByte((byte)part8);
+                            
+                            stream.WriteByte((byte)(neg ? ((sbyte)part8 | -0x80) : part8));
                             stream.WriteByte((byte)part0);
                         }
                         else
@@ -969,12 +977,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
                             {
                                 stream.WriteInt(len - 2);
 
-                                stream.WriteByte(ByteZero);
+                                stream.WriteByte((byte)(neg ? -0x80 : ByteZero));
+
+                                neg = false;
                             }
                             else
                                 stream.WriteInt(len - 3);
 
-                            stream.WriteByte((byte)part0);
+                            stream.WriteByte((byte)(neg ? ((sbyte)part0 | -0x80) : part0));
                         }
                     }
                     else
@@ -997,18 +1007,16 @@ public static void WriteDecimal(decimal val, IBinaryStream stream)
         {
             int scale = stream.ReadInt();
 
-            bool neg;
+            bool neg = false;
+
+            byte[] mag = ReadByteArray(stream);
 
-            if (scale < 0)
+            if ((sbyte)mag[0] < 0)
             {
-                scale = scale & 0x7FFFFFFF;
+                mag[0] &= 0x7F;
 
                 neg = true;
             }
-            else
-                neg = false;
-
-            byte[] mag = ReadByteArray(stream);
 
             if (scale < 0 || scale > 28)
                 throw new BinaryObjectException("Decimal value scale overflow (must be between 0 and 28): " + scale);

From 9e3a0ad5d6144525ef5b058bcf3e6429611bec1b Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Tue, 5 Sep 2017 19:36:59 +0300
Subject: [PATCH 355/516] IGNITE-4557 "Fixed wrong affinity manager call"

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../processors/cache/GridCacheAdapter.java    | 10 ++--
 .../cache/GridCacheAffinityManager.java       | 60 +++++++------------
 .../processors/cache/GridCacheContext.java    | 17 ------
 .../cache/GridCacheEvictionManager.java       |  6 +-
 .../processors/cache/GridCacheUtils.java      | 20 -------
 .../cache/affinity/GridCacheAffinityImpl.java | 16 ++---
 .../CacheDataStructuresManager.java           |  2 +-
 .../distributed/dht/GridDhtCacheAdapter.java  |  2 +-
 .../distributed/dht/GridDhtCacheEntry.java    |  2 +-
 .../dht/GridDhtLocalPartition.java            |  4 +-
 .../dht/GridDhtPartitionTopologyImpl.java     |  4 +-
 .../distributed/dht/GridDhtTxRemote.java      |  2 +-
 .../dht/GridPartitionedGetFuture.java         |  2 +-
 .../dht/GridPartitionedSingleGetFuture.java   |  2 +-
 .../dht/atomic/GridDhtAtomicCache.java        | 13 ++--
 .../GridNearAtomicSingleUpdateFuture.java     |  2 +-
 .../atomic/GridNearAtomicUpdateFuture.java    |  4 +-
 .../dht/colocated/GridDhtColocatedCache.java  |  6 +-
 .../colocated/GridDhtColocatedLockFuture.java |  4 +-
 .../preloader/GridDhtPartitionDemander.java   |  8 +--
 .../preloader/GridDhtPartitionSupplier.java   | 12 ++--
 .../dht/preloader/GridDhtPreloader.java       |  4 +-
 .../distributed/near/GridNearAtomicCache.java |  2 +-
 .../distributed/near/GridNearCacheEntry.java  |  6 +-
 .../distributed/near/GridNearGetFuture.java   |  4 +-
 .../distributed/near/GridNearLockFuture.java  |  2 +-
 ...OptimisticSerializableTxPrepareFuture.java |  2 +-
 .../GridNearOptimisticTxPrepareFuture.java    |  2 +-
 .../GridNearPessimisticTxPrepareFuture.java   |  2 +-
 .../near/GridNearTransactionalCache.java      |  6 +-
 .../near/GridNearTxFinishFuture.java          |  2 +-
 .../cache/query/GridCacheQueryManager.java    |  9 +--
 .../CacheContinuousQueryHandler.java          |  1 -
 .../CacheContinuousQueryManager.java          |  2 +-
 .../cache/transactions/IgniteTxAdapter.java   |  4 +-
 .../transactions/IgniteTxLocalAdapter.java    |  4 +-
 .../transactions/TxDeadlockDetection.java     |  2 +-
 .../datastreamer/DataStreamerImpl.java        |  2 +-
 .../datastructures/GridCacheSetImpl.java      |  2 +-
 .../datastructures/GridSetQueryPredicate.java |  2 +-
 .../processors/job/GridJobProcessor.java      |  2 +-
 .../cache/CacheAffinityCallSelfTest.java      |  4 +-
 .../GridCacheAbstractFullApiSelfTest.java     |  6 +-
 ...gniteCacheConfigVariationsFullApiTest.java |  6 +-
 .../IgniteCachePeekModesAbstractTest.java     |  8 +--
 ...tQueueFailoverDataConsistencySelfTest.java |  2 +-
 ...teCacheClientNodeChangingTopologyTest.java |  8 +--
 .../TxOptimisticDeadlockDetectionTest.java    |  2 +-
 .../TxPessimisticDeadlockDetectionTest.java   |  2 +-
 .../processors/query/h2/IgniteH2Indexing.java |  2 +-
 .../query/h2/opt/GridH2IndexBase.java         |  2 +-
 51 files changed, 126 insertions(+), 176 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index f0a263e74540c..aab0ed46fba15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -778,7 +778,7 @@ public String toString() {
                 boolean nearKey;
 
                 if (!(modes.near && modes.primary && modes.backup)) {
-                    boolean keyPrimary = ctx.affinity().primary(ctx.localNode(), part, topVer);
+                    boolean keyPrimary = ctx.affinity().primaryByPartition(ctx.localNode(), part, topVer);
 
                     if (keyPrimary) {
                         if (!modes.primary)
@@ -787,7 +787,7 @@ public String toString() {
                         nearKey = false;
                     }
                     else {
-                        boolean keyBackup = ctx.affinity().belongs(ctx.localNode(), part, topVer);
+                        boolean keyBackup = ctx.affinity().partitionBelongs(ctx.localNode(), part, topVer);
 
                         if (keyBackup) {
                             if (!modes.backup)
@@ -808,7 +808,7 @@ public String toString() {
                     }
                 }
                 else {
-                    nearKey = !ctx.affinity().belongs(ctx.localNode(), part, topVer);
+                    nearKey = !ctx.affinity().partitionBelongs(ctx.localNode(), part, topVer);
 
                     if (nearKey) {
                         // Swap and offheap are disabled for near cache.
@@ -3813,8 +3813,8 @@ IgniteInternalFuture<?> globalLoadCacheAsync(@Nullable IgniteBiPredicate<K, V> p
             /** {@inheritDoc} */
             @Override public boolean apply(ClusterNode clusterNode) {
                 return clusterNode.version().compareTo(PartitionSizeLongTask.SINCE_VER) >= 0 &&
-                    ((modes.primary && aff.primary(clusterNode, part, topVer)) ||
-                        (modes.backup && aff.backup(clusterNode, part, topVer)));
+                    ((modes.primary && aff.primaryByPartition(clusterNode, part, topVer)) ||
+                        (modes.backup && aff.backupByPartition(clusterNode, part, topVer)));
             }
         }).nodes();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
index d9ff61684e7d0..8d335fe46b397 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
@@ -238,8 +238,8 @@ public Object affinityKey(Object key) {
      * @param topVer Topology version.
      * @return Affinity nodes.
      */
-    public List<ClusterNode> nodes(Object key, AffinityTopologyVersion topVer) {
-        return nodes(partition(key), topVer);
+    public List<ClusterNode> nodesByKey(Object key, AffinityTopologyVersion topVer) {
+        return nodesByPartition(partition(key), topVer);
     }
 
     /**
@@ -247,7 +247,7 @@ public List<ClusterNode> nodes(Object key, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Affinity nodes.
      */
-    public List<ClusterNode> nodes(int part, AffinityTopologyVersion topVer) {
+    public List<ClusterNode> nodesByPartition(int part, AffinityTopologyVersion topVer) {
         if (cctx.isLocal())
             topVer = LOC_CACHE_TOP_VER;
 
@@ -282,8 +282,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Primary node for given key.
      */
-    @Nullable public ClusterNode primary(Object key, AffinityTopologyVersion topVer) {
-        return primary(partition(key), topVer);
+    @Nullable public ClusterNode primaryByKey(Object key, AffinityTopologyVersion topVer) {
+        return primaryByPartition(partition(key), topVer);
     }
 
     /**
@@ -291,8 +291,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return Primary node for given key.
      */
-    @Nullable public ClusterNode primary(int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    @Nullable public ClusterNode primaryByPartition(int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         if (nodes.isEmpty())
             return null;
@@ -306,8 +306,8 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code True} if checked node is primary for given key.
      */
-    public boolean primary(ClusterNode n, Object key, AffinityTopologyVersion topVer) {
-        return F.eq(primary(key, topVer), n);
+    public boolean primaryByKey(ClusterNode n, Object key, AffinityTopologyVersion topVer) {
+        return F.eq(primaryByKey(key, topVer), n);
     }
 
     /**
@@ -316,8 +316,8 @@ public boolean primary(ClusterNode n, Object key, AffinityTopologyVersion topVer
      * @param topVer Topology version.
      * @return {@code True} if checked node is primary for given partition.
      */
-    public boolean primary(ClusterNode n, int part, AffinityTopologyVersion topVer) {
-        return F.eq(primary(part, topVer), n);
+    public boolean primaryByPartition(ClusterNode n, int part, AffinityTopologyVersion topVer) {
+        return F.eq(primaryByPartition(part, topVer), n);
     }
 
     /**
@@ -325,8 +325,8 @@ public boolean primary(ClusterNode n, int part, AffinityTopologyVersion topVer)
      * @param topVer Topology version.
      * @return Backup nodes.
      */
-    public Collection<ClusterNode> backups(Object key, AffinityTopologyVersion topVer) {
-        return backups(partition(key), topVer);
+    public Collection<ClusterNode> backupsByKey(Object key, AffinityTopologyVersion topVer) {
+        return backupsByPartition(partition(key), topVer);
     }
 
     /**
@@ -334,8 +334,8 @@ public Collection<ClusterNode> backups(Object key, AffinityTopologyVersion topVe
      * @param topVer Topology version.
      * @return Backup nodes.
      */
-    public Collection<ClusterNode> backups(int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    private Collection<ClusterNode> backupsByPartition(int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         assert !F.isEmpty(nodes);
 
@@ -351,35 +351,21 @@ public Collection<ClusterNode> backups(int part, AffinityTopologyVersion topVer)
      * @param topVer Topology version.
      * @return {@code True} if checked node is a backup node for given partition.
      */
-    public boolean backup(ClusterNode n, int part, AffinityTopologyVersion topVer) {
-        List<ClusterNode> nodes = nodes(part, topVer);
+    public boolean backupByPartition(ClusterNode n, int part, AffinityTopologyVersion topVer) {
+        List<ClusterNode> nodes = nodesByPartition(part, topVer);
 
         assert !F.isEmpty(nodes);
 
         return nodes.indexOf(n) > 0;
     }
 
-    /**
-     * @param keys keys.
-     * @param topVer Topology version.
-     * @return Nodes for the keys.
-     */
-    public Collection<ClusterNode> remoteNodes(Iterable keys, AffinityTopologyVersion topVer) {
-        Collection<Collection<ClusterNode>> colcol = new GridLeanSet<>();
-
-        for (Object key : keys)
-            colcol.add(nodes(key, topVer));
-
-        return F.view(F.flatCollections(colcol), F.remoteNodes(cctx.localNodeId()));
-    }
-
     /**
      * @param key Key to check.
      * @param topVer Topology version.
      * @return {@code true} if given key belongs to local node.
      */
-    public boolean localNode(Object key, AffinityTopologyVersion topVer) {
-        return localNode(partition(key), topVer);
+    public boolean keyLocalNode(Object key, AffinityTopologyVersion topVer) {
+        return partitionLocalNode(partition(key), topVer);
     }
 
     /**
@@ -387,10 +373,10 @@ public boolean localNode(Object key, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code true} if given partition belongs to local node.
      */
-    public boolean localNode(int part, AffinityTopologyVersion topVer) {
+    public boolean partitionLocalNode(int part, AffinityTopologyVersion topVer) {
         assert part >= 0 : "Invalid partition: " + part;
 
-        return nodes(part, topVer).contains(cctx.localNode());
+        return nodesByPartition(part, topVer).contains(cctx.localNode());
     }
 
     /**
@@ -399,11 +385,11 @@ public boolean localNode(int part, AffinityTopologyVersion topVer) {
      * @param topVer Topology version.
      * @return {@code true} if given partition belongs to specified node.
      */
-    public boolean belongs(ClusterNode node, int part, AffinityTopologyVersion topVer) {
+    public boolean partitionBelongs(ClusterNode node, int part, AffinityTopologyVersion topVer) {
         assert node != null;
         assert part >= 0 : "Invalid partition: " + part;
 
-        return nodes(part, topVer).contains(node);
+        return nodesByPartition(part, topVer).contains(node);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 6322f9f03550a..3b44b5096f0b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -1589,23 +1589,6 @@ private void map(GridDhtCacheEntry entry, Iterable<ClusterNode> nodes,
         }
     }
 
-    /**
-     * Checks if at least one of the given keys belongs to one of the given partitions.
-     *
-     * @param keys Collection of keys to check.
-     * @param movingParts Collection of partitions to check against.
-     * @return {@code True} if there exist a key in collection {@code keys} that belongs
-     *      to one of partitions in {@code movingParts}
-     */
-    public boolean hasKey(Iterable<? extends K> keys, Collection<Integer> movingParts) {
-        for (K key : keys) {
-            if (movingParts.contains(affinity().partition(key)))
-                return true;
-        }
-
-        return false;
-    }
-
     /**
      * Check whether conflict resolution is required.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
index 134e743d4fd6e..f8722d6a4f338 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
@@ -808,7 +808,7 @@ public void touch(GridCacheEntryEx e, AffinityTopologyVersion topVer) {
             return;
 
         // Don't track non-primary entries if evicts are synchronized.
-        if (!cctx.isNear() && evictSync && !cctx.affinity().primary(cctx.localNode(), e.partition(), topVer))
+        if (!cctx.isNear() && evictSync && !cctx.affinity().primaryByPartition(cctx.localNode(), e.partition(), topVer))
             return;
 
         if (!busyLock.enterBusy())
@@ -910,7 +910,7 @@ public boolean evict(@Nullable GridCacheEntryEx entry, @Nullable GridCacheVersio
         if (evictSyncAgr) {
             assert !cctx.isNear(); // Make sure cache is not NEAR.
 
-            if (cctx.affinity().backups(
+            if (cctx.affinity().backupsByKey(
                     entry.key(),
                     cctx.topology().topologyVersion()).contains(cctx.localNode()) &&
                 evictSync)
@@ -1498,7 +1498,7 @@ void addEvent(DiscoveryEvent evt) {
                         if (!evts.isEmpty())
                             break;
 
-                        if (!cctx.affinity().primary(loc, it.next(), topVer))
+                        if (!cctx.affinity().primaryByPartition(loc, it.next(), topVer))
                             it.remove();
                     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 33d0085f3ff7c..af8f016a146f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -955,26 +955,6 @@ public static <K, V> void unwindEvicts(GridCacheSharedContext<K, V> ctx) {
             unwindEvicts(cacheCtx);
     }
 
-    /**
-     * Gets primary node on which given key is cached.
-     *
-     * @param ctx Cache.
-     * @param key Key to find primary node for.
-     * @return Primary node for the key.
-     */
-    @SuppressWarnings( {"unchecked"})
-    @Nullable public static ClusterNode primaryNode(GridCacheContext ctx, Object key) {
-        assert ctx != null;
-        assert key != null;
-
-        CacheConfiguration cfg = ctx.cache().configuration();
-
-        if (cfg.getCacheMode() != PARTITIONED)
-            return ctx.localNode();
-
-        return ctx.affinity().primary(key, ctx.affinity().affinityTopologyVersion());
-    }
-
     /**
      * @param asc {@code True} for ascending.
      * @return Descending order comparator.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
index 9e85bad610432..11361a27d384a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/affinity/GridCacheAffinityImpl.java
@@ -82,21 +82,21 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public boolean isPrimary(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().primary(n, key, topologyVersion());
+        return cctx.affinity().primaryByKey(n, key, topologyVersion());
     }
 
     /** {@inheritDoc} */
     @Override public boolean isBackup(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().backups(key, topologyVersion()).contains(n);
+        return cctx.affinity().backupsByKey(key, topologyVersion()).contains(n);
     }
 
     /** {@inheritDoc} */
     @Override public boolean isPrimaryOrBackup(ClusterNode n, K key) {
         A.notNull(n, "n", key, "key");
 
-        return cctx.affinity().belongs(n, cctx.affinity().partition(key), topologyVersion());
+        return cctx.affinity().partitionBelongs(n, cctx.affinity().partition(key), topologyVersion());
     }
 
     /** {@inheritDoc} */
@@ -126,7 +126,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
         AffinityTopologyVersion topVer = topologyVersion();
 
         for (int partsCnt = partitions(), part = 0; part < partsCnt; part++) {
-            for (ClusterNode affNode : cctx.affinity().nodes(part, topVer)) {
+            for (ClusterNode affNode : cctx.affinity().nodesByPartition(part, topVer)) {
                 if (n.id().equals(affNode.id())) {
                     parts.add(part);
 
@@ -142,7 +142,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public ClusterNode mapPartitionToNode(int part) {
         A.ensure(part >= 0 && part < partitions(), "part >= 0 && part < total partitions");
 
-        return F.first(cctx.affinity().nodes(part, topologyVersion()));
+        return F.first(cctx.affinity().nodesByPartition(part, topologyVersion()));
     }
 
     /** {@inheritDoc} */
@@ -204,7 +204,7 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
         Map<ClusterNode, Collection<K>> res = new HashMap<>(nodesCnt, 1.0f);
 
         for (K key : keys) {
-            ClusterNode primary = cctx.affinity().primary(key, topVer);
+            ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
             if (primary == null)
                 throw new IgniteException("Failed to get primary node [topVer=" + topVer + ", key=" + key + ']');
@@ -227,14 +227,14 @@ public GridCacheAffinityImpl(GridCacheContext<K, V> cctx) {
     @Override public Collection<ClusterNode> mapKeyToPrimaryAndBackups(K key) {
         A.notNull(key, "key");
 
-        return cctx.affinity().nodes(partition(key), topologyVersion());
+        return cctx.affinity().nodesByPartition(partition(key), topologyVersion());
     }
 
     /** {@inheritDoc} */
     @Override public Collection<ClusterNode> mapPartitionToPrimaryAndBackups(int part) {
         A.ensure(part >= 0 && part < partitions(), "part >= 0 && part < total partitions");
 
-        return cctx.affinity().nodes(part, topologyVersion());
+        return cctx.affinity().nodesByPartition(part, topologyVersion());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index 366a4a920b9ee..2b3080981ec36 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -455,7 +455,7 @@ private void removeSetData(IgniteUuid setId, AffinityTopologyVersion topVer) thr
         Collection<SetItemKey> keys = new ArrayList<>(BATCH_SIZE);
 
         for (SetItemKey key : set) {
-            if (!loc && !aff.primary(cctx.localNode(), key, topVer))
+            if (!loc && !aff.primaryByKey(cctx.localNode(), key, topVer))
                 continue;
 
             keys.add(key);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 6f95e0d6a1e7e..417eb3536536f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -934,7 +934,7 @@ public void sendTtlUpdateRequest(@Nullable final IgniteCacheExpiryPolicy expiryP
                     AffinityTopologyVersion topVer = ctx.shared().exchange().readyAffinityVersion();
 
                     for (Map.Entry<KeyCacheObject, GridCacheVersion> e : entries.entrySet()) {
-                        List<ClusterNode> nodes = ctx.affinity().nodes(e.getKey(), topVer);
+                        List<ClusterNode> nodes = ctx.affinity().nodesByKey(e.getKey(), topVer);
 
                         for (int i = 0; i < nodes.size(); i++) {
                             ClusterNode node = nodes.get(i);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
index cf4085ba02335..39571ff4720e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
@@ -402,7 +402,7 @@ public Collection<UUID> readers() throws GridCacheEntryRemovedException {
         }
 
         // If remote node is (primary?) or back up, don't add it as a reader.
-        if (cctx.affinity().belongs(node, partition(), topVer)) {
+        if (cctx.affinity().partitionBelongs(node, partition(), topVer)) {
             if (log.isDebugEnabled())
                 log.debug("Ignoring near reader because remote node is affinity node [locNodeId=" + cctx.localNodeId()
                     + ", rmtNodeId=" + nodeId + ", key=" + key + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index a0ccc28c33b0c..44d3f92160770 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -616,7 +616,7 @@ void onUnlock() {
      * @return {@code True} if local node is primary for this partition.
      */
     public boolean primary(AffinityTopologyVersion topVer) {
-        return cctx.affinity().primary(cctx.localNode(), id, topVer);
+        return cctx.affinity().primaryByPartition(cctx.localNode(), id, topVer);
     }
 
     /**
@@ -624,7 +624,7 @@ public boolean primary(AffinityTopologyVersion topVer) {
      * @return {@code True} if local node is backup for this partition.
      */
     public boolean backup(AffinityTopologyVersion topVer) {
-        return cctx.affinity().backup(cctx.localNode(), id, topVer);
+        return cctx.affinity().backupByPartition(cctx.localNode(), id, topVer);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 628db37ec3ef9..0cea80de02aac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -576,7 +576,7 @@ else if (!node2part.nodeId().equals(loc.id())) {
             for (int p = 0; p < num; p++) {
                 GridDhtLocalPartition locPart = localPartition(p, topVer, false, false);
 
-                if (cctx.affinity().localNode(p, topVer)) {
+                if (cctx.affinity().partitionLocalNode(p, topVer)) {
                     // This partition will be created during next topology event,
                     // which obviously has not happened at this point.
                     if (locPart == null) {
@@ -699,7 +699,7 @@ private GridDhtLocalPartition localPartition(int p,
         try {
             loc = locParts.get(p);
 
-            boolean belongs = cctx.affinity().localNode(p, topVer);
+            boolean belongs = cctx.affinity().partitionLocalNode(p, topVer);
 
             if (loc != null && loc.state() == EVICTED) {
                 locParts.set(p, loc = null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
index 8942ef9d9178c..399736e8f4278 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
@@ -286,7 +286,7 @@ IgniteUuid remoteFutureId() {
             return true;
 
         // Check if we are on the backup node.
-        return !cacheCtx.affinity().backups(key, topVer).contains(cctx.localNode());
+        return !cacheCtx.affinity().backupsByKey(key, topVer).contains(cctx.localNode());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index c41711c245028..519239aecb532 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -380,7 +380,7 @@ private boolean map(
     ) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index 2b5624b489b6f..a3f6b72376e8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -325,7 +325,7 @@ private void map(AffinityTopologyVersion topVer) {
     @Nullable private ClusterNode mapKeyToNode(AffinityTopologyVersion topVer) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 8a8221b428f13..ffe68ed2cfee7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -2490,7 +2490,7 @@ private UpdateSingleResult updateSingle(
 
                 assert !(newConflictVer instanceof GridCacheVersionEx) : newConflictVer;
 
-                boolean primary = !req.fastMap() || ctx.affinity().primary(ctx.localNode(), entry.partition(),
+                boolean primary = !req.fastMap() || ctx.affinity().primaryByPartition(ctx.localNode(), entry.partition(),
                     req.topologyVersion());
 
                 Object writeVal = op == TRANSFORM ? req.entryProcessor(i) : req.writeValue(i);
@@ -2580,7 +2580,7 @@ else if (conflictCtx.isMerge())
 
                 if (hasNear) {
                     if (primary && updRes.sendToDht()) {
-                        if (!ctx.affinity().belongs(node, entry.partition(), topVer)) {
+                        if (!ctx.affinity().partitionBelongs(node, entry.partition(), topVer)) {
                             // If put the same value as in request then do not need to send it back.
                             if (op == TRANSFORM || writeVal != updRes.newValue()) {
                                 res.addNearValue(i,
@@ -2711,7 +2711,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
                 Map<KeyCacheObject, CacheObject> storeMap = req.fastMap() ?
                     F.view(putMap, new P1<CacheObject>() {
                         @Override public boolean apply(CacheObject key) {
-                            return ctx.affinity().primary(ctx.localNode(), key, req.topologyVersion());
+                            return ctx.affinity().primaryByKey(ctx.localNode(), key, req.topologyVersion());
                         }
                     }) :
                     putMap;
@@ -2734,7 +2734,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
                 Collection<KeyCacheObject> storeKeys = req.fastMap() ?
                     F.view(rmvKeys, new P1<Object>() {
                         @Override public boolean apply(Object key) {
-                            return ctx.affinity().primary(ctx.localNode(), key, req.topologyVersion());
+                            return ctx.affinity().primaryByKey(ctx.localNode(), key, req.topologyVersion());
                         }
                     }) :
                     rmvKeys;
@@ -2773,7 +2773,8 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
 
                     assert writeVal != null || op == DELETE : "null write value found.";
 
-                    boolean primary = !req.fastMap() || ctx.affinity().primary(ctx.localNode(), entry.key(),
+                    boolean primary = !req.fastMap() || ctx.affinity().primaryByPartition(ctx.localNode(),
+                        entry.partition(),
                         req.topologyVersion());
 
                     Collection<UUID> readers = null;
@@ -2869,7 +2870,7 @@ else if (F.contains(readers, node.id())) // Reader became primary or backup.
 
                     if (hasNear) {
                         if (primary) {
-                            if (!ctx.affinity().belongs(node, entry.partition(), topVer)) {
+                            if (!ctx.affinity().partitionBelongs(node, entry.partition(), topVer)) {
                                 int idx = firstEntryIdx + i;
 
                                 if (req.operation() == TRANSFORM) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
index 7376affca4455..891a20c2884a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
@@ -543,7 +543,7 @@ private GridNearAtomicAbstractUpdateRequest mapSingleUpdate(AffinityTopologyVers
         else
             val = EntryProcessorResourceInjectorProxy.wrap(cctx.kernalContext(), (EntryProcessor)val);
 
-        ClusterNode primary = cctx.affinity().primary(cacheKey, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(cacheKey, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to map keys for cache (all partition nodes " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index a252d9ae30c2b..9bdd1becb7600 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -937,7 +937,7 @@ else if (conflictRmvVals != null) {
         else
             val = EntryProcessorResourceInjectorProxy.wrap(cctx.kernalContext(), (EntryProcessor)val);
 
-        ClusterNode primary = cctx.affinity().primary(cacheKey.partition(), topVer);
+        ClusterNode primary = cctx.affinity().primaryByPartition(cacheKey.partition(), topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to map keys for cache (all partition nodes " +
@@ -988,7 +988,7 @@ private List<ClusterNode> mapKey(KeyCacheObject key, AffinityTopologyVersion top
 
         // If we can send updates in parallel - do it.
         return fastMap ? cctx.topology().nodes(affMgr.partition(key), topVer) :
-            Collections.singletonList(affMgr.primary(key, topVer));
+            Collections.singletonList(affMgr.primaryByKey(key, topVer));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index c9fc983d4850c..e1e0ec2bf087e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -174,7 +174,7 @@ public GridDistributedCacheEntry entryExx(
         AffinityTopologyVersion topVer,
         boolean allowDetached
     ) {
-        return allowDetached && !ctx.affinity().primary(ctx.localNode(), key, topVer) ?
+        return allowDetached && !ctx.affinity().primaryByKey(ctx.localNode(), key, topVer) ?
             createEntry(key) : entryExx(key, topVer);
     }
 
@@ -670,7 +670,7 @@ else if (!skipVals && ctx.config().isStatisticsEnabled())
                     assert topVer.compareTo(AffinityTopologyVersion.ZERO) > 0;
 
                     // Send request to remove from remote nodes.
-                    ClusterNode primary = ctx.affinity().primary(key, topVer);
+                    ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                     if (primary == null) {
                         if (log.isDebugEnabled())
@@ -790,7 +790,7 @@ public void removeLocks(long threadId, GridCacheVersion ver, Collection<KeyCache
                         map = U.newHashMap(affNodes.size());
                     }
 
-                    ClusterNode primary = ctx.affinity().primary(key, topVer);
+                    ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                     if (primary == null) {
                         if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index 69b66f9bfdcd3..79ca1087a3b88 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -1203,7 +1203,7 @@ private boolean mapAsPrimary(Collection<KeyCacheObject> keys, AffinityTopologyVe
         boolean explicit = false;
 
         for (KeyCacheObject key : keys) {
-            if (!cctx.affinity().primary(cctx.localNode(), key, topVer)) {
+            if (!cctx.affinity().primaryByKey(cctx.localNode(), key, topVer)) {
                 // Remove explicit locks added so far.
                 for (KeyCacheObject k : keys)
                     cctx.mvcc().removeExplicitLock(threadId, cctx.txKey(k), lockVer);
@@ -1287,7 +1287,7 @@ private GridNearLockMapping map(
     ) throws IgniteCheckedException {
         assert mapping == null || mapping.node() != null;
 
-        ClusterNode primary = cctx.affinity().primary(key, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to lock keys " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 260533d948e0e..c2b0c1390def7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -625,7 +625,7 @@ public void handleSupplyMessage(
             for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
                 int p = e.getKey();
 
-                if (cctx.affinity().localNode(p, topVer)) {
+                if (cctx.affinity().partitionLocalNode(p, topVer)) {
                     GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                     assert part != null;
@@ -695,7 +695,7 @@ public void handleSupplyMessage(
 
             // Only request partitions based on latest topology version.
             for (Integer miss : supply.missed()) {
-                if (cctx.affinity().localNode(miss, topVer))
+                if (cctx.affinity().partitionLocalNode(miss, topVer))
                     fut.partitionMissed(id, miss);
             }
 
@@ -1386,7 +1386,7 @@ private void demandFromNode(
                         for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
                             int p = e.getKey();
 
-                            if (cctx.affinity().localNode(p, topVer)) {
+                            if (cctx.affinity().partitionLocalNode(p, topVer)) {
                                 GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                                 assert part != null;
@@ -1463,7 +1463,7 @@ private void demandFromNode(
 
                         // Only request partitions based on latest topology version.
                         for (Integer miss : s.supply().missed()) {
-                            if (cctx.affinity().localNode(miss, topVer))
+                            if (cctx.affinity().partitionLocalNode(miss, topVer))
                                 fut.partitionMissed(node.id(), miss);
                         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index b082c4736cb9c..994242354c05d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -302,7 +302,7 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
                             (Iterator<GridCacheMapEntry>)sctx.entryIt : loc.allEntries().iterator();
 
                         while (entIt.hasNext()) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition, so we send '-1' partition and move on.
                                 s.missed(part);
 
@@ -387,7 +387,7 @@ else if (log.isDebugEnabled())
                             boolean prepared = false;
 
                             while (iter.hasNext()) {
-                                if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                                if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                     // Demander no longer needs this partition,
                                     // so we send '-1' partition and move on.
                                     s.missed(part);
@@ -510,7 +510,7 @@ else if (log.isDebugEnabled())
                             (Iterator<GridCacheEntryInfo>)sctx.entryIt : entries.iterator();
 
                         while (lsnrIt.hasNext()) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition,
                                 // so we send '-1' partition and move on.
                                 s.missed(part);
@@ -808,7 +808,7 @@ private void processOldDemandMessage(GridDhtPartitionDemandMessage d, UUID id) {
                     boolean partMissing = false;
 
                     for (GridCacheEntryEx e : loc.allEntries()) {
-                        if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                        if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                             // Demander no longer needs this partition, so we send '-1' partition and move on.
                             s.missed(part);
 
@@ -859,7 +859,7 @@ else if (log.isDebugEnabled())
                                 boolean prepared = false;
 
                                 for (Map.Entry<byte[], GridCacheSwapEntry> e : iter) {
-                                    if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                                    if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                         // Demander no longer needs this partition,
                                         // so we send '-1' partition and move on.
                                         s.missed(part);
@@ -947,7 +947,7 @@ else if (log.isDebugEnabled())
                         swapLsnr = null;
 
                         for (GridCacheEntryInfo info : entries) {
-                            if (!cctx.affinity().belongs(node, part, d.topologyVersion())) {
+                            if (!cctx.affinity().partitionBelongs(node, part, d.topologyVersion())) {
                                 // Demander no longer needs this partition,
                                 // so we send '-1' partition and move on.
                                 s.missed(part);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 2efaed859b714..2d2e7a8154a69 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -289,7 +289,7 @@ private IgniteCheckedException stopError() {
             }
 
             // If partition belongs to local node.
-            if (cctx.affinity().localNode(p, topVer)) {
+            if (cctx.affinity().partitionLocalNode(p, topVer)) {
                 GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
                 assert part != null;
@@ -349,7 +349,7 @@ private IgniteCheckedException stopError() {
      * @return Picked owners.
      */
     private Collection<ClusterNode> pickedOwners(int p, AffinityTopologyVersion topVer) {
-        Collection<ClusterNode> affNodes = cctx.affinity().nodes(p, topVer);
+        Collection<ClusterNode> affNodes = cctx.affinity().nodesByPartition(p, topVer);
 
         int affCnt = affNodes.size();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index b843e4e6d3db1..41632ef338b58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -161,7 +161,7 @@ public void processNearAtomicUpdateResponse(
             if (F.contains(failed, key))
                 continue;
 
-            if (ctx.affinity().belongs(ctx.localNode(), ctx.affinity().partition(key), req.topologyVersion())) { // Reader became backup.
+            if (ctx.affinity().partitionBelongs(ctx.localNode(), ctx.affinity().partition(key), req.topologyVersion())) { // Reader became backup.
                 GridCacheEntryEx entry = peekEx(key);
 
                 if (entry != null && entry.markObsolete(ver))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index 30fc213fc63e2..d022805a76bf2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -112,7 +112,7 @@ public GridNearCacheEntry(
                 return false;
             }
 
-            if (cctx.affinity().backup(cctx.localNode(), part, topVer)) {
+            if (cctx.affinity().backupByPartition(cctx.localNode(), part, topVer)) {
                 this.topVer = AffinityTopologyVersion.NONE;
 
                 return false;
@@ -162,7 +162,7 @@ public void initializeFromDht(AffinityTopologyVersion topVer) throws GridCacheEn
                                 }
                             }
 
-                            ClusterNode primaryNode = cctx.affinity().primary(key, topVer);
+                            ClusterNode primaryNode = cctx.affinity().primaryByKey(key, topVer);
 
                             if (primaryNode == null)
                                 this.topVer = AffinityTopologyVersion.NONE;
@@ -686,7 +686,7 @@ private void primaryNode(UUID nodeId, AffinityTopologyVersion topVer) {
         ClusterNode primary = null;
 
         try {
-            primary = cctx.affinity().primary(part, topVer);
+            primary = cctx.affinity().primaryByPartition(part, topVer);
         }
         catch (IllegalStateException ignore) {
             // Do not have affinity history.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index cb47498be0167..fb2843c02eec9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -413,7 +413,7 @@ private Map<KeyCacheObject, GridNearCacheEntry> map(
     ) {
         int part = cctx.affinity().partition(key);
 
-        List<ClusterNode> affNodes = cctx.affinity().nodes(part, topVer);
+        List<ClusterNode> affNodes = cctx.affinity().nodesByPartition(part, topVer);
 
         if (affNodes.isEmpty()) {
             onDone(serverNotFoundError(topVer));
@@ -726,7 +726,7 @@ private Map<K, V> loadEntries(
                     info.unmarshalValue(cctx, cctx.deploy().globalLoader());
 
                     // Entries available locally in DHT should not be loaded into near cache for reading.
-                    if (!cctx.affinity().localNode(info.key(), cctx.affinity().affinityTopologyVersion())) {
+                    if (!cctx.affinity().keyLocalNode(info.key(), cctx.affinity().affinityTopologyVersion())) {
                         GridNearCacheEntry entry = savedEntries.get(info.key());
 
                         if (entry == null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index d7a0fb5f8213d..d3e3a15831bb2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -1373,7 +1373,7 @@ private GridNearLockMapping map(
     ) throws IgniteCheckedException {
         assert mapping == null || mapping.node() != null;
 
-        ClusterNode primary = cctx.affinity().primary(key, topVer);
+        ClusterNode primary = cctx.affinity().primaryByKey(key, topVer);
 
         if (primary == null)
             throw new ClusterTopologyServerNotFoundException("Failed to lock keys " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index c464b36217fd8..a8448dcfd9680 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -529,7 +529,7 @@ private void map(
         GridCacheContext cacheCtx = entry.context();
 
         List<ClusterNode> nodes = cacheCtx.isLocal() ?
-            cacheCtx.affinity().nodes(entry.key(), topVer) :
+            cacheCtx.affinity().nodesByKey(entry.key(), topVer) :
             cacheCtx.topology().nodes(cacheCtx.affinity().partition(entry.key()), topVer);
 
         txMapping.addMapping(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index b314b81bf7b50..606d70ffb17fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -605,7 +605,7 @@ private GridDistributedTxMapping map(
             nodes = cacheCtx.topology().nodes(cached0.partition(), topVer);
         else
             nodes = cacheCtx.isLocal() ?
-                cacheCtx.affinity().nodes(entry.key(), topVer) :
+                cacheCtx.affinity().nodesByKey(entry.key(), topVer) :
                 cacheCtx.topology().nodes(cacheCtx.affinity().partition(entry.key()), topVer);
 
         txMapping.addMapping(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index f9a2f90787b3c..a4132f22b2b6b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -196,7 +196,7 @@ private void preparePessimistic() {
             GridCacheContext cacheCtx = txEntry.context();
 
             List<ClusterNode> nodes = cacheCtx.isLocal() ?
-                cacheCtx.affinity().nodes(txEntry.key(), topVer) :
+                cacheCtx.affinity().nodesByKey(txEntry.key(), topVer) :
                 cacheCtx.topology().nodes(cacheCtx.affinity().partition(txEntry.key()), topVer);
 
             ClusterNode primary = F.first(nodes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
index b3eb7551f05a0..940dd809779a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
@@ -476,7 +476,7 @@ private void processLockResponse(UUID nodeId, GridNearLockResponse res) {
      * @return {@code True} if entry is locally mapped as a primary or back up node.
      */
     protected boolean isNearLocallyMapped(GridCacheEntryEx e, AffinityTopologyVersion topVer) {
-        return ctx.affinity().belongs(ctx.localNode(), e.partition(), topVer);
+        return ctx.affinity().partitionBelongs(ctx.localNode(), e.partition(), topVer);
     }
 
     /**
@@ -548,7 +548,7 @@ protected boolean evictNearEntry(GridCacheEntryEx e, GridCacheVersion obsoleteVe
                             topVer = cand.topologyVersion();
 
                             // Send request to remove from remote nodes.
-                            ClusterNode primary = ctx.affinity().primary(key, topVer);
+                            ClusterNode primary = ctx.affinity().primaryByKey(key, topVer);
 
                             if (primary == null) {
                                 if (log.isDebugEnabled())
@@ -668,7 +668,7 @@ public void removeLocks(GridCacheVersion ver, Collection<KeyCacheObject> keys) {
                                     map = U.newHashMap(affNodes.size());
                                 }
 
-                                ClusterNode primary = ctx.affinity().primary(key, cand.topologyVersion());
+                                ClusterNode primary = ctx.affinity().primaryByKey(key, cand.topologyVersion());
 
                                 if (primary == null) {
                                     if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 9acab56ae2d8e..aed1ab0ac18de 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -350,7 +350,7 @@ else if (err != null)
                             GridCacheContext cacheCtx = e.context();
 
                             try {
-                                if (e.op() != NOOP && !cacheCtx.affinity().localNode(e.key(), topVer)) {
+                                if (e.op() != NOOP && !cacheCtx.affinity().keyLocalNode(e.key(), topVer)) {
                                     GridCacheEntryEx entry = cacheCtx.cache().peekEx(e.key());
 
                                     if (entry != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 7efb746d4428e..47f1bed98680c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -1580,11 +1580,11 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                     // Other types are filtered in indexing manager.
                     if (!cctx.isReplicated() && qry.type() == SCAN && qry.partition() == null &&
                         cctx.config().getCacheMode() != LOCAL && !incBackups &&
-                        !cctx.affinity().primary(cctx.localNode(), key, topVer)) {
+                        !cctx.affinity().primaryByKey(cctx.localNode(), key, topVer)) {
                         if (log.isDebugEnabled())
                             log.debug("Ignoring backup element [row=" + row +
                                 ", cacheMode=" + cctx.config().getCacheMode() + ", incBackups=" + incBackups +
-                                ", primary=" + cctx.affinity().primary(cctx.localNode(), key, topVer) + ']');
+                                ", primary=" + cctx.affinity().primaryByKey(cctx.localNode(), key, topVer) + ']');
 
                         continue;
                     }
@@ -1592,7 +1592,8 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                     V val = row.getValue();
 
                     if (log.isDebugEnabled()) {
-                        ClusterNode primaryNode = CU.primaryNode(cctx, key);
+                        ClusterNode primaryNode = cctx.affinity().primaryByKey(key,
+                            cctx.affinity().affinityTopologyVersion());
 
                         log.debug(S.toString("Record",
                             "key", key, true,
@@ -2355,7 +2356,7 @@ public Collection<GridCacheSqlMetadata> sqlMetadata() throws IgniteCheckedExcept
 
                 return new IgniteBiPredicate<K, V>() {
                     @Override public boolean apply(K k, V v) {
-                        return cache.context().affinity().primary(ctx.discovery().localNode(), k, NONE);
+                        return cache.context().affinity().primaryByKey(ctx.discovery().localNode(), k, NONE);
                     }
                 };
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 26c361498fbab..ceebbe96dbd56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -56,7 +56,6 @@
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheDeploymentManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 9c300e70adab6..acd5bf1a454a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -372,7 +372,7 @@ public void onEntryExpired(GridCacheEntryEx e, KeyCacheObject key, CacheObject o
         if (F.isEmpty(lsnrCol))
             return;
 
-        boolean primary = cctx.affinity().primary(cctx.localNode(), e.partition(), AffinityTopologyVersion.NONE);
+        boolean primary = cctx.affinity().primaryByPartition(cctx.localNode(), e.partition(), AffinityTopologyVersion.NONE);
 
         if (cctx.isReplicated() || primary) {
             boolean recordIgniteEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index cd5babe7118b0..266c5a83cd8c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -1293,7 +1293,7 @@ protected void batchStoreCommit(Iterable<IgniteTxEntry> writeEntries) throws Ign
                         if (!skip && skipNonPrimary) {
                             skip = e.cached().isNear() ||
                                 e.cached().detached() ||
-                                !e.context().affinity().primary(e.cached().partition(), topologyVersion()).isLocal();
+                                !e.context().affinity().primaryByPartition(e.cached().partition(), topologyVersion()).isLocal();
                         }
 
                         if (!skip && !local() && // Update local store at backups only if needed.
@@ -1712,7 +1712,7 @@ protected boolean isNearLocallyMapped(IgniteTxEntry e, boolean primaryOnly) {
 
         int part = cached != null ? cached.partition() : cacheCtx.affinity().partition(e.key());
 
-        List<ClusterNode> affNodes = cacheCtx.affinity().nodes(part, topologyVersion());
+        List<ClusterNode> affNodes = cacheCtx.affinity().nodesByPartition(part, topologyVersion());
 
         e.locallyMapped(F.contains(affNodes, cctx.localNode()));
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 7f161859da101..0d992286114b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1106,7 +1106,7 @@ assert isWriteToStoreFromDhtValid(stores) :
      * @return {@code True} if local node is current primary for given entry.
      */
     private boolean primaryLocal(GridCacheEntryEx entry) {
-        return entry.context().affinity().primary(cctx.localNode(), entry.partition(), AffinityTopologyVersion.NONE);
+        return entry.context().affinity().primaryByPartition(cctx.localNode(), entry.partition(), AffinityTopologyVersion.NONE);
     }
 
     /**
@@ -1407,7 +1407,7 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
                     finally {
                         if (entry != null && readCommitted()) {
                             if (cacheCtx.isNear()) {
-                                if (cacheCtx.affinity().belongs(cacheCtx.localNode(), entry.partition(), topVer)) {
+                                if (cacheCtx.affinity().partitionBelongs(cacheCtx.localNode(), entry.partition(), topVer)) {
                                     if (entry.markObsolete(xidVer))
                                         cacheCtx.cache().removeEntry(entry);
                                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
index 70d938e144ead..67d00ea782923 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetection.java
@@ -401,7 +401,7 @@ private void mapTxKeys(@Nullable Set<IgniteTxKey> txKeys, Map<IgniteTxKey, TxLoc
         private UUID primary(IgniteTxKey txKey) {
             GridCacheContext ctx = cctx.cacheContext(txKey.cacheId());
 
-            ClusterNode node = ctx.affinity().primary(txKey.key(), topVer);
+            ClusterNode node = ctx.affinity().primaryByKey(txKey.key(), topVer);
 
             assert node != null : topVer;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index 8978fbf34dffe..604b35d85a91e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -1951,7 +1951,7 @@ else if (ttl == CU.TTL_NOT_CHANGED)
                         expiryTime = CU.toExpireTime(ttl);
                     }
 
-                    boolean primary = cctx.affinity().primary(cctx.localNode(), entry.key(), topVer);
+                    boolean primary = cctx.affinity().primaryByKey(cctx.localNode(), entry.key(), topVer);
 
                     entry.initialValue(e.getValue(),
                         ver,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
index 70232afe717fa..d5f277ebf3f16 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
@@ -496,7 +496,7 @@ private Collection<ClusterNode> dataNodes(AffinityTopologyVersion topVer) throws
         Collection<ClusterNode> nodes;
 
         if (collocated) {
-            List<ClusterNode> nodes0 = ctx.affinity().nodes(hdrPart, topVer);
+            List<ClusterNode> nodes0 = ctx.affinity().nodesByPartition(hdrPart, topVer);
 
             nodes = !nodes0.isEmpty() ?
                 Collections.singleton(nodes0.contains(ctx.localNode()) ? ctx.localNode() : F.first(nodes0)) : nodes0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
index e8b2cc7f0f4b8..bc6c1827a0774 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
@@ -91,7 +91,7 @@ public IgniteUuid setId() {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public boolean apply(K k, V v) {
-        return !filter || ctx.affinity().primary(ctx.localNode(), k, ctx.affinity().affinityTopologyVersion());
+        return !filter || ctx.affinity().primaryByKey(ctx.localNode(), k, ctx.affinity().affinityTopologyVersion());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index 7889a5579c7bb..992e02a6fc570 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -1568,7 +1568,7 @@ public PartitionsReservation(int[] cacheIds, int partId,
                         }
                     }
                     finally {
-                        if (checkPartMapping && !cctx.affinity().primary(partId, topVer).id().equals(ctx.localNodeId()))
+                        if (checkPartMapping && !cctx.affinity().primaryByPartition(partId, topVer).id().equals(ctx.localNodeId()))
                             throw new IgniteException("Failed partition reservation. " +
                                 "Partition is not primary on the node. [partition=" + partId + ", cacheName=" + cctx.name() +
                                 ", nodeId=" + ctx.localNodeId() + ", topology=" + topVer + ']');
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
index 92e2b9b2ba21d..b0337d62967e0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheAffinityCallSelfTest.java
@@ -214,12 +214,12 @@ public CheckCallable(Object key, AffinityTopologyVersion topVer) {
 
                 ClusterNode loc = ignite.cluster().localNode();
 
-                if (loc.equals(aff.primary(key, topVer)))
+                if (loc.equals(aff.primaryByKey(key, topVer)))
                     return true;
 
                 AffinityTopologyVersion topVer0 = new AffinityTopologyVersion(topVer.topologyVersion() + 1, 0);
 
-                assertEquals(loc, aff.primary(key, topVer0));
+                assertEquals(loc, aff.primaryByKey(key, topVer0));
             }
 
             return null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
index 1cfb330da8cd6..53b4900410033 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
@@ -5837,7 +5837,7 @@ public CheckEntriesTask(Collection<String> keys) {
             int size = 0;
 
             for (String key : keys) {
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx())) {
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx())) {
                     GridCacheEntryEx e =
                         ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key);
 
@@ -5873,7 +5873,7 @@ private static class CheckCacheSizeTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : map.keySet())
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(ctx.name()).localSize(ALL));
@@ -6116,7 +6116,7 @@ public CheckKeySizeTask(Collection<String> keys) {
             int size = 0;
 
             for (String key : keys)
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(null).localSize(ALL));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
index 6b0e1932e1f2c..d4449f9ee9228 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java
@@ -5548,7 +5548,7 @@ private static class CheckEntriesTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : keys) {
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx())) {
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx())) {
                     GridCacheEntryEx e =
                         ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key);
 
@@ -5589,7 +5589,7 @@ private static class CheckCacheSizeTask extends TestIgniteIdxRunnable {
             int size = 0;
 
             for (String key : map.keySet())
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(ctx.name()).localSize(ALL));
@@ -5850,7 +5850,7 @@ public CheckKeySizeTask(Collection<String> keys, String s) {
             int size = 0;
 
             for (String key : keys)
-                if (ctx.affinity().localNode(key, ctx.discovery().topologyVersionEx()))
+                if (ctx.affinity().keyLocalNode(key, ctx.discovery().topologyVersionEx()))
                     size++;
 
             assertEquals("Incorrect key size on cache #" + idx, size, ignite.cache(cacheName).localSize(ALL));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
index 6c577c63ab693..fac24ccd00045 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
@@ -1009,9 +1009,9 @@ private T2<Integer, Integer> swapKeysCount(int nodeIdx, int part) throws IgniteC
         //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primary(ctx.localNode(), part, topVer))
+        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             primaryCnt = cnt;
-        else if (affinity.backup(ctx.localNode(), part, topVer))
+        else if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             backupCnt = cnt;
         return new T2<>(primaryCnt, backupCnt);
     }
@@ -1081,9 +1081,9 @@ private T2<Integer, Integer> offheapKeysCount(int nodeIdx, int part) throws Igni
         //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primary(ctx.localNode(), part, topVer))
+        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
             primaryCnt = cnt;
-        else if (affinity.backup(ctx.localNode(), part, topVer))
+        else if (affinity.backupByPartition(ctx.localNode(), part, topVer))
             backupCnt = cnt;
         return new T2<>(primaryCnt, backupCnt);
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
index 45b4b9f299aa3..aeca2fb2279ff 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheAbstractQueueFailoverDataConsistencySelfTest.java
@@ -365,7 +365,7 @@ private int primaryQueueNode(IgniteQueue queue) {
 
         for (int i = 0; i < gridCount(); i++) {
             for (GridCacheEntryEx e : ((IgniteKernal)grid(i)).context().cache().internalCache(cctx.name()).allEntries()) {
-                if (aff.primary(grid(i).localNode(), e.key(), AffinityTopologyVersion.NONE)
+                if (aff.primaryByKey(grid(i).localNode(), e.key(), AffinityTopologyVersion.NONE)
                     && e.key().value(cctx.cacheObjectContext(), false) instanceof GridCacheQueueHeaderKey)
                     return i;
             }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
index b4ef11a82c0c1..8709d05664d94 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java
@@ -853,13 +853,13 @@ public void testPessimisticTx2() throws Exception {
 
         GridCacheAffinityManager aff = ignite0.context().cache().internalCache(null).context().affinity();
 
-        List<ClusterNode> nodes1 = aff.nodes(key1, topVer1);
-        List<ClusterNode> nodes2 = aff.nodes(key1, topVer2);
+        List<ClusterNode> nodes1 = aff.nodesByKey(key1, topVer1);
+        List<ClusterNode> nodes2 = aff.nodesByKey(key1, topVer2);
 
         assertEquals(nodes1, nodes2);
 
-        nodes1 = aff.nodes(key2, topVer1);
-        nodes2 = aff.nodes(key2, topVer2);
+        nodes1 = aff.nodesByKey(key2, topVer1);
+        nodes2 = aff.nodesByKey(key2, topVer2);
 
         assertFalse(nodes1.get(0).equals(nodes2.get(0)));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
index f6a06c29a9fbe..66cfc4ee7338c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
@@ -289,7 +289,7 @@ private void doTestDeadlock(
                     key = keys.get(1);
 
                     ClusterNode primaryNode =
-                        ((IgniteCacheProxy)cache).context().affinity().primary(key, NONE);
+                        ((IgniteCacheProxy)cache).context().affinity().primaryByKey(key, NONE);
 
                     List<Integer> primaryKeys =
                         primaryKeys(grid(primaryNode).cache(CACHE_NAME), 5, key + (100 * threadNum));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
index 83eb908d21978..ced8b61166876 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
@@ -293,7 +293,7 @@ private void doTestDeadlock(
                     key = keys.get(1);
 
                     ClusterNode primaryNode =
-                        ((IgniteCacheProxy)cache).context().affinity().primary(key, NONE);
+                        ((IgniteCacheProxy)cache).context().affinity().primaryByKey(key, NONE);
 
                     List<Integer> primaryKeys =
                         primaryKeys(grid(primaryNode).cache(CACHE_NAME), 5, key + (100 * threadNum));
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 66fb7ae94f5be..e4b0c1feb7a21 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -2106,7 +2106,7 @@ private void createSqlFunctions(String schema, Class<?>[] clss) throws IgniteChe
 
                 return new IgniteBiPredicate<K, V>() {
                     @Override public boolean apply(K k, V v) {
-                        return aff.primary(locNode, k, topVer0);
+                        return aff.primaryByKey(locNode, k, topVer0);
                     }
                 };
             }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 22b94c7ac796b..3700774260ded 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -612,7 +612,7 @@ private ClusterNode rangeNode(GridCacheContext<?,?> cctx, GridH2QueryContext qct
             node = cctx.discovery().node(nodeId);
         }
         else // Get primary node for current topology version.
-            node = cctx.affinity().primary(affKeyObj, qctx.topologyVersion());
+            node = cctx.affinity().primaryByKey(affKeyObj, qctx.topologyVersion());
 
         if (node == null) // Node was not found, probably topology changed and we need to retry the whole query.
             throw new GridH2RetryException("Failed to find node.");

From 61e2809fa3a65b96d283a44de4c9e42c4dae1a0d Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 6 Sep 2017 11:54:19 +0300
Subject: [PATCH 356/516] IGNITE-6219 - IgniteCache#loadCache executes local
 load in caller thread

(cherry picked from commit 0e63f59)
---
 .../processors/task/GridTaskWorker.java       |  22 +++-
 .../cache/store/GridStoreLoadCacheTest.java   | 120 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite.java      |   5 +-
 3 files changed, 145 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/GridStoreLoadCacheTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
index c426008f5fb11..8787f4b92b9bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
@@ -579,7 +579,7 @@ private void processMappedJobs(Map<? extends ComputeJob, ClusterNode> jobs) thro
         if (F.isEmpty(jobs))
             return;
 
-        Collection<GridJobResultImpl> jobResList = new ArrayList<>(jobs.size());
+        List<GridJobResultImpl> jobResList = new ArrayList<>(jobs.size());
 
         Collection<ComputeJobSibling> sibs = new ArrayList<>(jobs.size());
 
@@ -637,6 +637,26 @@ private void processMappedJobs(Map<? extends ComputeJob, ClusterNode> jobs) thro
         // Set mapped flag.
         ses.onMapped();
 
+        // Move local jobs to the end of the list, because
+        // they will be invoked in current thread that will hold other
+        // jobs.
+        int jobResSize = jobResList.size();
+
+        if (jobResSize > 1) {
+            UUID locId = ctx.discovery().localNode().id();
+
+            for (int i = 0; i < jobResSize; i++) {
+                UUID jobNodeId = jobResList.get(i).getNode().id();
+
+                if (jobNodeId.equals(locId) && i < jobResSize - 1) {
+                    Collections.swap(jobResList, i, jobResSize - 1);
+
+                    jobResSize--;
+                    i--;
+                }
+            }
+        }
+
         // Send out all remote mappedJobs.
         for (GridJobResultImpl res : jobResList) {
             evtLsnr.onJobSend(this, res.getSibling());
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/GridStoreLoadCacheTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/GridStoreLoadCacheTest.java
new file mode 100644
index 0000000000000..d88c4318ee27e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/GridStoreLoadCacheTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store;
+
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.lang.IgniteBiInClosure;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test checks that local cacheLoad task never blocks remote
+ * cacheLoad.
+ */
+public class GridStoreLoadCacheTest extends GridCommonAbstractTest {
+    /** Barrier. */
+    private static final CyclicBarrier BARRIER = new CyclicBarrier(3);
+
+    /** Cache name. */
+    public static final String CACHE_NAME = "test";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        //noinspection unchecked
+        cfg.setCacheConfiguration(new CacheConfiguration(CACHE_NAME).setCacheStoreFactory(new TestFactory()));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void test() throws Exception {
+        for (int i = 0; i < 3; i++) {
+            IgniteEx srv1 = startGrid(0);
+            startGrid(1);
+            startGrid(2);
+
+            awaitPartitionMapExchange();
+
+            srv1.cache(CACHE_NAME).loadCache(null);
+
+            stopAllGrids();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestFactory implements Factory<CacheStore> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public CacheStore create() {
+            return new TestStore();
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestStore extends CacheStoreAdapter<Object, Object> {
+        /** {@inheritDoc} */
+        @Override public void loadCache(IgniteBiInClosure<Object, Object> clo, Object... args) {
+            try {
+                BARRIER.await(3, TimeUnit.SECONDS);
+            }
+            catch (InterruptedException | BrokenBarrierException | TimeoutException e) {
+                throw new IgniteException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object load(Object key) throws CacheLoaderException {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<?, ?> entry) throws CacheWriterException {
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            // No-op
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index a24f020103e54..995a2023b8de9 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.cache.affinity.local.LocalAffinityFunctionTest;
 import org.apache.ignite.cache.store.GridCacheBalancingStoreSelfTest;
 import org.apache.ignite.cache.store.GridCacheLoadOnlyStoreAdapterSelfTest;
+import org.apache.ignite.cache.store.GridStoreLoadCacheTest;
 import org.apache.ignite.cache.store.StoreResourceInjectionSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest;
@@ -40,12 +41,12 @@
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreTest;
 import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreMultithreadedSelfTest;
 import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreSelfTest;
+import org.apache.ignite.cache.store.jdbc.JdbcTypesDefaultTransformerTest;
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceMultipleConnectionsTest;
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalancePairedConnectionsTest;
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceTest;
 import org.apache.ignite.internal.managers.communication.IgniteIoTestMessagesTest;
 import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest;
-import org.apache.ignite.cache.store.jdbc.JdbcTypesDefaultTransformerTest;
 import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest;
 import org.apache.ignite.internal.processors.cache.CacheEntryProcessorCopySelfTest;
@@ -346,6 +347,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(IgniteCommunicationBalanceMultipleConnectionsTest.class);
         suite.addTestSuite(IgniteIoTestMessagesTest.class);
 
+        suite.addTestSuite(GridStoreLoadCacheTest.class);
+
         return suite;
     }
 }

From 5b1fdfaee7422c02177513f6f8c1365a262fd49b Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 6 Sep 2017 18:33:31 +0300
Subject: [PATCH 357/516] IGNITE-6256: DiscoCache should always contains local
 node.

---
 .../managers/discovery/GridDiscoveryManager.java     | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 40dea984ee56d..8a625fc728da3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -613,7 +613,7 @@ else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
                     topHist.clear();
 
                     topSnap.set(new Snapshot(AffinityTopologyVersion.ZERO,
-                        createDiscoCache(locNode, Collections.<ClusterNode>emptySet())));
+                        createDiscoCache(locNode, Collections.<ClusterNode>singleton(locNode))));
                 }
                 else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                     assert locNode.isClient() : locNode;
@@ -1969,6 +1969,8 @@ public void reconnect() {
      * @return Newly created discovery cache.
      */
     @NotNull private DiscoCache createDiscoCache(ClusterNode loc, Collection<ClusterNode> topSnapshot) {
+        assert topSnapshot.contains(loc);
+
         HashSet<UUID> alives = U.newHashSet(topSnapshot.size());
         HashMap<UUID, ClusterNode> nodeMap = U.newHashMap(topSnapshot.size());
 
@@ -2172,15 +2174,15 @@ public void scheduleSegmentCheck() {
                     lastChk = now;
 
                     if (!segValid) {
-                        List<ClusterNode> empty = Collections.emptyList();
-
                         ClusterNode node = getSpi().getLocalNode();
 
+                        Collection<ClusterNode> locNodeOnlyTopology = Collections.singleton(node);
+
                         discoWrk.addEvent(EVT_NODE_SEGMENTED,
                             AffinityTopologyVersion.NONE,
                             node,
-                            createDiscoCache(node, empty),
-                            empty,
+                            createDiscoCache(node, locNodeOnlyTopology),
+                            locNodeOnlyTopology,
                             null);
 
                         lastSegChkRes.set(false);

From e1f1a51670bff70c4cd906505e0df6fa85b7a38c Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 5 Sep 2017 19:11:12 +0300
Subject: [PATCH 358/516] GG-12699: Fix
 GridCacheAbstractFullApiSelfTest.testTransformResourceInjection.

Squashed commit of the following:

commit 3f20fe8dcc796406f7a7791e3ae9ddb5c26183ca
Author: Nikolay Izhikov <nizhikov.dev@gmail.com>
Date:   Wed Aug 9 13:37:11 2017 +0300

    IGNITE-5897 Fix session init/end logic. This fixes tests.

    Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>

    (cherry picked from commit 5a559df)

commit 52e89d387874a0653c58a608cc000950a76fb6b0
Author: dpavlov <dpavlov@gridgain.com>
Date:   Wed Jul 26 17:23:05 2017 +0300

    IGNITE-5806 - Fixed assertion with a side-effect - Fixes #2335.

    Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

    (cherry picked from commit 9e79c4b)

commit d3c40e418dce6ab640fe06e8c18ada4b93f1edf5
Author: Andrey V. Mashenkov <andrey.mashenkov@gmail.com>
Date:   Mon Sep 4 15:57:16 2017 +0300

    Fix javadoc.
---
 .../cache/store/CacheStoreManager.java        |  4 +++-
 .../store/GridCacheStoreManagerAdapter.java   | 23 +++++++++----------
 .../cache/transactions/IgniteTxAdapter.java   |  9 ++++++--
 .../resource/GridResourceProcessor.java       |  6 ++++-
 4 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
index 67c9334fa6ed2..459c7029fbde8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
@@ -165,9 +165,11 @@ public boolean removeAll(@Nullable IgniteInternalTx tx, Collection<Object> keys)
     /**
      * @param tx Transaction.
      * @param commit Commit.
+     * @param last {@code True} if this is last store in transaction.
+     * @param storeSessionEnded {@code True} if session for underlying store already ended.
      * @throws IgniteCheckedException If failed.
      */
-    public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException;
+    public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last, boolean storeSessionEnded) throws IgniteCheckedException;
 
     /**
      * End session initiated by write-behind store.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index c35b8fb9d2c01..6ab8c56578593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -777,7 +777,8 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException {
+    @Override public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last,
+        boolean storeSessionEnded) throws IgniteCheckedException {
         assert store != null;
 
         sessionInit0(tx);
@@ -788,7 +789,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
                     lsnr.onSessionEnd(locSes, commit);
             }
 
-            if (!sesHolder.get().ended(store))
+            if (!sesHolder.get().ended(store) && !storeSessionEnded)
                 store.sessionEnd(commit);
         }
         catch (Throwable e) {
@@ -855,7 +856,7 @@ private void sessionInit0(@Nullable IgniteInternalTx tx) {
 
         sesHolder.set(ses);
 
-        if (sesLsnrs != null && !ses.started(this)) {
+        if (!ses.started(store) && sesLsnrs != null) {
             for (CacheStoreSessionListener lsnr : sesLsnrs)
                 lsnr.onSessionStart(locSes);
         }
@@ -872,7 +873,8 @@ private void sessionEnd0(@Nullable IgniteInternalTx tx, boolean threwEx) throws
                         lsnr.onSessionEnd(locSes, !threwEx);
                 }
 
-                store.sessionEnd(!threwEx);
+                if (!sesHolder.get().ended(store))
+                    store.sessionEnd(!threwEx);
             }
         }
         catch (Exception e) {
@@ -914,11 +916,8 @@ private static class SessionData {
         private Object attachment;
 
         /** */
-        private final Set<CacheStoreManager> started =
-            new GridSetWrapper<>(new IdentityHashMap<CacheStoreManager, Object>());
-
-        /** */
-        private final Set<CacheStore> ended = new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
+        private final Set<CacheStore> started =
+            new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
 
         /**
          * @param tx Current transaction.
@@ -981,8 +980,8 @@ private void cacheName(String cacheName) {
         /**
          * @return If session is started.
          */
-        private boolean started(CacheStoreManager mgr) {
-            return !started.add(mgr);
+        private boolean started(CacheStore store) {
+            return !started.add(store);
         }
 
         /**
@@ -990,7 +989,7 @@ private boolean started(CacheStoreManager mgr) {
          * @return Whether session already ended on this store instance.
          */
         private boolean ended(CacheStore store) {
-            return !ended.add(store);
+            return !started.remove(store);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index b07a1175c436d..266c5a83cd8c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -40,6 +41,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -61,6 +63,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
+import org.apache.ignite.internal.util.GridSetWrapper;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
 import org.apache.ignite.internal.util.lang.GridTuple;
@@ -1238,13 +1241,15 @@ protected boolean isWriteToStoreFromDhtValid(Collection<CacheStoreManager> store
      * @param commit Commit flag.
      * @throws IgniteCheckedException In case of error.
      */
-    protected void sessionEnd(Collection<CacheStoreManager> stores, boolean commit) throws IgniteCheckedException {
+    protected void sessionEnd(final Collection<CacheStoreManager> stores, boolean commit) throws IgniteCheckedException {
         Iterator<CacheStoreManager> it = stores.iterator();
 
+        Set<CacheStore> visited = new GridSetWrapper<>(new IdentityHashMap<CacheStore, Object>());
+
         while (it.hasNext()) {
             CacheStoreManager store = it.next();
 
-            store.sessionEnd(this, commit, !it.hasNext());
+            store.sessionEnd(this, commit, !it.hasNext(), !visited.add(store.store()));
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
index 84d07b64e50e5..02616b5e983a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
@@ -20,7 +20,6 @@
 import java.lang.annotation.Annotation;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.util.Collection;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.store.CacheStoreSession;
 import org.apache.ignite.compute.ComputeJob;
@@ -318,6 +317,11 @@ private GridResourceInjector injectorByAnnotation(GridResourceIoc.ResourceAnnota
 
     /**
      * @param obj Object to inject.
+     * @param ann Annotation enum.
+     * @param dep Grid deployment object.
+     * @param depCls Grid deployment class.
+     * @param param Resource to inject.
+     * @return {@code True} if resource was injected.
      * @throws IgniteCheckedException If failed to inject.
      */
     private boolean inject(Object obj, GridResourceIoc.ResourceAnnotation ann, @Nullable GridDeployment dep,

From c746277c2c6ce07871ba29fa3cd1bab7a78f8662 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Tue, 5 Sep 2017 18:04:54 +0300
Subject: [PATCH 359/516] ignite-5145 Support multiple service deployment in
 API

(cherry picked from commit 0b6da97)
---
 .../org/apache/ignite/IgniteServices.java     |  57 +-
 .../ignite/internal/IgniteServicesImpl.java   |  25 +-
 .../GridServiceDeploymentCompoundFuture.java  | 196 +++++
 .../service/GridServiceProcessor.java         | 481 ++++++++---
 .../service/PreparedConfigurations.java       |  53 ++
 .../service/ServiceDeploymentException.java   |  78 ++
 .../util/future/GridCompoundFuture.java       |  15 +-
 .../util/future/GridFutureAdapter.java        |   2 +-
 ...rviceDeploymentCompoundFutureSelfTest.java | 241 ++++++
 ...idServiceProcessorBatchDeploySelfTest.java | 765 ++++++++++++++++++
 .../testsuites/IgniteKernalSelfTestSuite.java |   4 +
 11 files changed, 1786 insertions(+), 131 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
index 8365ec7e9523b..ae3958fc269cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
@@ -20,6 +20,7 @@
 import java.util.Collection;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.service.ServiceDeploymentException;
 import org.apache.ignite.lang.IgniteAsyncSupport;
 import org.apache.ignite.lang.IgniteAsyncSupported;
 import org.apache.ignite.resources.IgniteInstanceResource;
@@ -154,10 +155,10 @@ public interface IgniteServices extends IgniteAsyncSupport {
      *
      * @param name Service name.
      * @param svc Service instance.
-     * @throws IgniteException If failed to deploy service.
+     * @throws ServiceDeploymentException If failed to deploy service.
      */
     @IgniteAsyncSupported
-    public void deployClusterSingleton(String name, Service svc) throws IgniteException;
+    public void deployClusterSingleton(String name, Service svc) throws ServiceDeploymentException;
 
     /**
      * Deploys a per-node singleton service. Ignite will guarantee that there is always
@@ -170,10 +171,10 @@ public interface IgniteServices extends IgniteAsyncSupport {
      *
      * @param name Service name.
      * @param svc Service instance.
-     * @throws IgniteException If failed to deploy service.
+     * @throws ServiceDeploymentException If failed to deploy service.
      */
     @IgniteAsyncSupported
-    public void deployNodeSingleton(String name, Service svc) throws IgniteException;
+    public void deployNodeSingleton(String name, Service svc) throws ServiceDeploymentException;
 
     /**
      * Deploys one instance of this service on the primary node for a given affinity key.
@@ -204,11 +205,11 @@ public interface IgniteServices extends IgniteAsyncSupport {
      * @param cacheName Name of the cache on which affinity for key should be calculated, {@code null} for
      *      default cache.
      * @param affKey Affinity cache key.
-     * @throws IgniteException If failed to deploy service.
+     * @throws ServiceDeploymentException If failed to deploy service.
      */
     @IgniteAsyncSupported
     public void deployKeyAffinitySingleton(String name, Service svc, @Nullable String cacheName, Object affKey)
-        throws IgniteException;
+        throws ServiceDeploymentException;
 
     /**
      * Deploys multiple instances of the service on the grid. Ignite will deploy a
@@ -238,10 +239,11 @@ public void deployKeyAffinitySingleton(String name, Service svc, @Nullable Strin
      * @param svc Service instance.
      * @param totalCnt Maximum number of deployed services in the grid, {@code 0} for unlimited.
      * @param maxPerNodeCnt Maximum number of deployed services on each node, {@code 0} for unlimited.
-     * @throws IgniteException If failed to deploy service.
+     * @throws ServiceDeploymentException If failed to deploy service.
      */
     @IgniteAsyncSupported
-    public void deployMultiple(String name, Service svc, int totalCnt, int maxPerNodeCnt) throws IgniteException;
+    public void deployMultiple(String name, Service svc, int totalCnt, int maxPerNodeCnt)
+        throws ServiceDeploymentException;
 
     /**
      * Deploys multiple instances of the service on the grid according to provided
@@ -277,10 +279,31 @@ public void deployKeyAffinitySingleton(String name, Service svc, @Nullable Strin
      * </pre>
      *
      * @param cfg Service configuration.
-     * @throws IgniteException If failed to deploy service.
+     * @throws ServiceDeploymentException If failed to deploy service.
      */
     @IgniteAsyncSupported
-    public void deploy(ServiceConfiguration cfg) throws IgniteException;
+    public void deploy(ServiceConfiguration cfg) throws ServiceDeploymentException;
+
+    /**
+     * Deploys multiple services described by provided configurations. Depending on specified parameters, multiple
+     * instances of the same service may be deployed (see {@link ServiceConfiguration}).
+     * Whenever topology changes, Ignite will automatically rebalance
+     * the deployed services within cluster to make sure that each node will end up with
+     * about equal number of deployed instances whenever possible.
+     *
+     * If deployment fails, then {@link ServiceDeploymentException} containing a list of failed services will be
+     * thrown. It is guaranteed that all services that were provided to this method and are not present in the list of
+     * failed services are successfully deployed by the moment of the exception being thrown.
+     *
+     * @param cfgs {@link Collection} of service configurations to be deployed.
+     * @param allOrNone Specifies behavior in case when errors during deployment occur. If {@code true}, then two
+     * outcomes are possible: either all services will be deployed, or none of them. If {@code false}, then partial
+     * deployments are permitted.
+     * @throws ServiceDeploymentException If failed to deploy services.
+     * @see IgniteServices#deploy(ServiceConfiguration)
+     */
+    @IgniteAsyncSupported
+    public void deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) throws ServiceDeploymentException;
 
     /**
      * Cancels service deployment. If a service with specified name was deployed on the grid,
@@ -298,6 +321,20 @@ public void deployKeyAffinitySingleton(String name, Service svc, @Nullable Strin
     @IgniteAsyncSupported
     public void cancel(String name) throws IgniteException;
 
+    /**
+     * Cancels services with specified names.
+     * <p>
+     * Note that depending on user logic, it may still take extra time for a service to
+     * finish execution, even after it was cancelled.
+     * <p>
+     * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+     *
+     * @param names Names of services to cancel.
+     * @throws IgniteException If failed to cancel services.
+     */
+    @IgniteAsyncSupported
+    public void cancelAll(Collection<String> names) throws IgniteException;
+
     /**
      * Cancels all deployed services.
      * <p>
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
index df6e5df4fc411..a51f3f1ba95fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
@@ -23,6 +23,7 @@
 import java.io.ObjectOutput;
 import java.io.ObjectStreamException;
 import java.util.Collection;
+import java.util.Collections;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteServices;
@@ -150,10 +151,17 @@ public IgniteServicesImpl(GridKernalContext ctx, ClusterGroupAdapter prj, boolea
     @Override public void deploy(ServiceConfiguration cfg) {
         A.notNull(cfg, "cfg");
 
+        deployAll(Collections.singleton(cfg), false);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+        A.notNull(cfgs, "cfgs");
+
         guard();
 
         try {
-            saveOrGet(ctx.service().deploy(cfg));
+            saveOrGet(ctx.service().deployAll(cfgs, allOrNone));
         }
         catch (IgniteCheckedException e) {
             throw U.convertException(e);
@@ -180,6 +188,21 @@ public IgniteServicesImpl(GridKernalContext ctx, ClusterGroupAdapter prj, boolea
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void cancelAll(Collection<String> names) {
+        guard();
+
+        try {
+            saveOrGet(ctx.service().cancelAll(names));
+        }
+        catch (IgniteCheckedException e) {
+            throw U.convertException(e);
+        }
+        finally {
+            unguard();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void cancelAll() {
         guard();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
new file mode 100644
index 0000000000000..12b88e5f119f5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Service deployment compound future, {@code allOrNone} parameter specifies failing policy.
+ * <p>
+ * If {@code allOrNone} parameter is set to {@code false}, then this future waits for completion of all child futures.
+ * If any exceptions are thrown during deployment, then {@link IgniteCheckedException} with {@link
+ * ServiceDeploymentException} as a cause will be thrown from {@link IgniteInternalFuture#get get()} method after all
+ * futures complete or fail. Inner exception will contain configurations of failed services.
+ */
+public class GridServiceDeploymentCompoundFuture extends GridCompoundFuture<Object, Object> {
+    /** */
+    private final boolean allOrNone;
+
+    /** Kernal context. */
+    private final GridKernalContext ctx;
+
+    /** Logger. */
+    private final IgniteLogger log;
+
+    /** Names of services written to cache during current deployment. */
+    private Collection<String> svcsToRollback;
+
+    /** */
+    private volatile ServiceDeploymentException err;
+
+    /**
+     * @param allOrNone Failing policy.
+     * @param ctx Kernal context.
+     */
+    GridServiceDeploymentCompoundFuture(boolean allOrNone, GridKernalContext ctx) {
+        this.allOrNone = allOrNone;
+        this.ctx = ctx;
+        this.log = ctx.log(getClass());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean processFailure(Throwable err, IgniteInternalFuture<Object> fut) {
+        assert fut instanceof GridServiceDeploymentFuture : fut;
+
+        GridServiceDeploymentFuture depFut = (GridServiceDeploymentFuture)fut;
+
+        if (allOrNone) {
+            if (initialized()) {
+                onDone(new IgniteCheckedException(
+                    new ServiceDeploymentException("Failed to deploy provided services.", err, getConfigurations())));
+            }
+            else {
+                synchronized (this) {
+                    if (this.err == null) {
+                        this.err = new ServiceDeploymentException("Failed to deploy provided services.", err,
+                            new ArrayList<ServiceConfiguration>());
+                    }
+                    else
+                        this.err.addSuppressed(err);
+                }
+            }
+        }
+        else {
+            synchronized (this) {
+                if (this.err == null)
+                    this.err = new ServiceDeploymentException("Failed to deploy some services.",
+                        new ArrayList<ServiceConfiguration>());
+
+                this.err.getFailedConfigurations().add(depFut.configuration());
+                this.err.addSuppressed(err);
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Marks this future as initialized. Will complete with error if failures before initialization occurred and
+     * all-or-none policy is followed.
+     */
+    public void serviceDeploymentMarkInitialized() {
+        if (allOrNone && this.err != null) {
+            this.err.getFailedConfigurations().addAll(getConfigurations());
+
+            onDone(new IgniteCheckedException(this.err));
+        }
+        else
+            super.markInitialized();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean onDone(@Nullable final Object res, @Nullable Throwable err, final boolean cancel) {
+        final Throwable resErr;
+
+        if (err == null && this.err != null)
+            resErr = new IgniteCheckedException(this.err);
+        else
+            resErr = err;
+
+        if (allOrNone && this.err != null && svcsToRollback != null) {
+            U.warn(log, "Failed to deploy provided services. The following services will be cancelled:" + svcsToRollback);
+
+            IgniteInternalFuture<?> fut = ctx.service().cancelAll(svcsToRollback);
+
+            /*
+            Can not call fut.get() since it is possible we are in system pool now and
+            fut also should be completed from system pool.
+             */
+            fut.listen(new IgniteInClosure<IgniteInternalFuture>() {
+                @Override public void apply(IgniteInternalFuture fut) {
+                    try {
+                        fut.get();
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to cancel deployed services.", e);
+                    }
+                    finally {
+                        svcsToRollback = null;
+                    }
+
+                    GridServiceDeploymentCompoundFuture.super.onDone(res, resErr, cancel);
+                }
+            });
+
+            return false;
+        }
+
+        return super.onDone(res, resErr, cancel);
+    }
+
+    /**
+     * @param fut Child future.
+     * @param own If {@code true}, then corresponding service will be cancelled on failure.
+     */
+    public void add(GridServiceDeploymentFuture fut, boolean own) {
+        super.add(fut);
+
+        if (own) {
+            if (svcsToRollback == null)
+                svcsToRollback = new ArrayList<>();
+
+            svcsToRollback.add(fut.configuration().getName());
+        }
+    }
+
+    /**
+     * @return Collection of names of services that were written to cache during current deployment.
+     */
+    public Collection<String> servicesToRollback() {
+        if (svcsToRollback != null)
+            return svcsToRollback;
+        else
+            return Collections.emptyList();
+    }
+
+    /**
+     * @return Collection of configurations, stored in child futures.
+     */
+    private Collection<ServiceConfiguration> getConfigurations() {
+        Collection<IgniteInternalFuture<Object>> futs = futures();
+
+        List<ServiceConfiguration> cfgs = new ArrayList<>(futs.size());
+
+        for (IgniteInternalFuture<Object> fut : futs)
+            cfgs.add(((GridServiceDeploymentFuture)fut).configuration());
+
+        return cfgs;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index c961eb0443cdc..f457d8091a4ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.service;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -92,6 +93,7 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.plugin.security.SecurityException;
 import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
@@ -101,6 +103,7 @@
 import org.apache.ignite.services.ServiceDescriptor;
 import org.apache.ignite.spi.IgniteNodeValidationResult;
 import org.apache.ignite.thread.IgniteThreadFactory;
+import org.apache.ignite.transactions.Transaction;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
@@ -111,6 +114,7 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.UTILITY_CACHE_NAME;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
 import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
 /**
@@ -294,19 +298,13 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         ServiceConfiguration[] cfgs = ctx.config().getServiceConfiguration();
 
         if (cfgs != null) {
-            Collection<IgniteInternalFuture<?>> futs = new ArrayList<>();
-
             for (ServiceConfiguration c : cfgs) {
                 // Deploy only on server nodes by default.
                 if (c.getNodeFilter() == null)
                     c.setNodeFilter(ctx.cluster().get().forServers().predicate());
-
-                futs.add(deploy(c));
             }
 
-            // Await for services to deploy.
-            for (IgniteInternalFuture<?> f : futs)
-                f.get();
+            deployAll(Arrays.asList(cfgs), true).get();
         }
 
         if (log.isDebugEnabled())
@@ -487,130 +485,276 @@ public IgniteInternalFuture<?> deployKeyAffinitySingleton(String name, Service s
     }
 
     /**
-     * @param cfg Service configuration.
-     * @return Future for deployment.
+     * @param cfgs Service configurations.
+     * @param allOrNone Failure processing policy.
+     * @return Configurations to deploy.
      */
-    public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
-        A.notNull(cfg, "cfg");
+    private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+        List<ServiceConfiguration> cfgsCp = new ArrayList<>(cfgs.size());
 
         ServicesCompatibilityState state = markCompatibilityStateAsUsed();
 
-        validate(cfg);
-
-        ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
+        Marshaller marsh = ctx.config().getMarshaller();
 
-        if (!state.srvcCompatibility) {
-            Marshaller marsh = ctx.config().getMarshaller();
+        List<GridServiceDeploymentFuture> failedFuts = null;
 
-            LazyServiceConfiguration cfg0;
+        for (ServiceConfiguration cfg : cfgs) {
+            Exception err = null;
 
             try {
-                byte[] srvcBytes = U.marshal(marsh, cfg.getService());
-
-                cfg0 = new LazyServiceConfiguration(cfg, srvcBytes);
+                validate(cfg);
             }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to marshal service with configured marshaller [srvc=" + cfg.getService()
-                    + ", marsh=" + marsh + "]", e);
+            catch (Exception e) {
+                U.error(log, "Failed to validate service configuration [name=" + cfg.getName() +
+                    ", srvc=" + cfg.getService() + ']', e);
 
-                return new GridFinishedFuture<>(e);
+                err = e;
             }
 
-            cfg = cfg0;
-        }
+            if (err == null) {
+                try {
+                    ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
+                }
+                catch (Exception e) {
+                    U.error(log, "Failed to authorize service creation [name=" + cfg.getName() +
+                        ", srvc=" + cfg.getService() + ']', e);
 
-        GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+                    err = e;
+                }
+            }
 
-        GridServiceDeploymentFuture old = depFuts.putIfAbsent(cfg.getName(), fut);
+            if (err == null) {
+                if (!state.srvcCompatibility) {
+                    try {
+                        byte[] srvcBytes = U.marshal(marsh, cfg.getService());
 
-        if (old != null) {
-            if (!old.configuration().equalsIgnoreNodeFilter(cfg)) {
-                fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
-                    "different configuration) [deployed=" + old.configuration() + ", new=" + cfg + ']'));
+                        cfgsCp.add(new LazyServiceConfiguration(cfg, srvcBytes));
+                    }
+                    catch (Exception e) {
+                        U.error(log, "Failed to marshal service with configured marshaller [name=" + cfg.getName() +
+                            ", srvc=" + cfg.getService() + ", marsh=" + marsh + "]", e);
 
-                return fut;
+                        err = e;
+                    }
+                }
+                else
+                    cfgsCp.add(cfg);
             }
 
-            return old;
-        }
+            if (err != null) {
+                if (allOrNone) {
+                    return new PreparedConfigurations(null,
+                        null,
+                        new IgniteCheckedException(
+                            new ServiceDeploymentException("None of the provided services were deplyed.", err, cfgs)));
+                }
+                else {
+                    if (failedFuts == null)
+                        failedFuts = new ArrayList<>();
 
-        if (ctx.clientDisconnected()) {
-            fut.onDone(new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
-                "Failed to deploy service, client node disconnected."));
+                    GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+
+                    fut.onDone(err);
 
-            depFuts.remove(cfg.getName(), fut);
+                    failedFuts.add(fut);
+                }
+            }
         }
 
-        while (true) {
-            try {
-                GridServiceDeploymentKey key = new GridServiceDeploymentKey(cfg.getName());
+        return new PreparedConfigurations(cfgsCp, failedFuts, null);
+    }
 
-                if (ctx.deploy().enabled())
-                    ctx.cache().context().deploy().ignoreOwnership(true);
+    /**
+     * @param cfgs Service configurations.
+     * @param allOrNone Failure processing policy.
+     * @return Future for deployment.
+     */
+    public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+        assert cfgs != null;
 
-                try {
-                    GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
-                        new GridServiceDeployment(ctx.localNodeId(), cfg));
+        PreparedConfigurations srvCfg = prepareServiceConfigurations(cfgs, allOrNone);
 
-                    if (dep != null) {
-                        if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
-                            // Remove future from local map.
-                            depFuts.remove(cfg.getName(), fut);
+        if (srvCfg.err != null)
+            return new GridFinishedFuture<>(srvCfg.err);
 
-                            fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
-                                "different configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']'));
-                        }
-                        else {
-                            Iterator<Cache.Entry<Object, Object>> it = serviceEntries(
-                                ServiceAssignmentsPredicate.INSTANCE);
+        List<ServiceConfiguration> cfgsCp = srvCfg.cfgs;
+
+        List<GridServiceDeploymentFuture> failedFuts = srvCfg.failedFuts;
+
+        Collections.sort(cfgsCp, new Comparator<ServiceConfiguration>() {
+            @Override public int compare(ServiceConfiguration cfg1, ServiceConfiguration cfg2) {
+                return cfg1.getName().compareTo(cfg2.getName());
+            }
+        });
 
-                            while (it.hasNext()) {
-                                Cache.Entry<Object, Object> e = it.next();
+        GridServiceDeploymentCompoundFuture res;
 
-                                if (e.getKey() instanceof GridServiceAssignmentsKey) {
-                                    GridServiceAssignments assigns = (GridServiceAssignments)e.getValue();
+        while (true) {
+            res = new GridServiceDeploymentCompoundFuture(allOrNone, ctx);
 
-                                    if (assigns.name().equals(cfg.getName())) {
-                                        // Remove future from local map.
-                                        depFuts.remove(cfg.getName(), fut);
+            if (ctx.deploy().enabled())
+                ctx.cache().context().deploy().ignoreOwnership(true);
 
-                                        fut.onDone();
+            try {
+                if (cfgsCp.size() == 1)
+                    writeServiceToCache(res, cfgsCp.get(0));
+                else if (cfgsCp.size() > 1) {
+                    try (Transaction tx = cache.txStart(PESSIMISTIC, READ_COMMITTED)) {
+                        for (ServiceConfiguration cfg : cfgsCp) {
+                            try {
+                                writeServiceToCache(res, cfg);
+                            }
+                            catch (IgniteCheckedException e) {
+                                if (X.hasCause(e, ClusterTopologyCheckedException.class))
+                                    throw e; // Retry.
 
-                                        break;
-                                    }
+                                if (allOrNone) {
+                                    for (String name : res.servicesToRollback())
+                                        depFuts.remove(name).onDone(e);
+
+                                    res.onDone(new IgniteCheckedException(new ServiceDeploymentException(
+                                        "Failed to deploy provided services.", e, cfgs)));
+
+                                    return res;
                                 }
                             }
-
-                            if (!dep.configuration().equalsIgnoreNodeFilter(cfg))
-                                U.warn(log, "Service already deployed with different configuration (will ignore) " +
-                                    "[deployed=" + dep.configuration() + ", new=" + cfg + ']');
                         }
+
+                        tx.commit();
                     }
                 }
-                finally {
-                    if (ctx.deploy().enabled())
-                        ctx.cache().context().deploy().ignoreOwnership(false);
+
+                break;
+            }
+            catch (IgniteException | IgniteCheckedException e) {
+                for (String name : res.servicesToRollback())
+                    depFuts.remove(name).onDone(e);
+
+                if (X.hasCause(e, ClusterTopologyCheckedException.class)) {
+                    if (log.isDebugEnabled())
+                        log.debug("Topology changed while deploying services (will retry): " + e.getMessage());
                 }
+                else {
+                    res.onDone(new IgniteCheckedException(
+                        new ServiceDeploymentException("Failed to deploy provided services.", e, cfgs)));
 
-                return fut;
+                    return res;
+                }
             }
-            catch (ClusterTopologyCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
+            finally {
+                if (ctx.deploy().enabled())
+                    ctx.cache().context().deploy().ignoreOwnership(false);
             }
-            catch (IgniteCheckedException e) {
-                if (e.hasCause(ClusterTopologyCheckedException.class)) {
-                    if (log.isDebugEnabled())
-                        log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
+        }
 
-                    continue;
+        if (ctx.clientDisconnected()) {
+            IgniteClientDisconnectedCheckedException err =
+                new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
+                "Failed to deploy services, client node disconnected: " + cfgs);
+
+            for (String name : res.servicesToRollback()) {
+                GridServiceDeploymentFuture fut = depFuts.remove(name);
+
+                if (fut != null)
+                    fut.onDone(err);
+            }
+
+            return new GridFinishedFuture<>(err);
+        }
+
+        if (failedFuts != null) {
+            for (GridServiceDeploymentFuture fut : failedFuts)
+                res.add(fut, false);
+        }
+
+        res.serviceDeploymentMarkInitialized();
+
+        return res;
+    }
+
+    /**
+     * @param res Resulting compound future.
+     * @param cfg Service configuration.
+     * @throws IgniteCheckedException If operation failed.
+     */
+    private void writeServiceToCache(GridServiceDeploymentCompoundFuture res, ServiceConfiguration cfg)
+        throws IgniteCheckedException {
+        String name = cfg.getName();
+
+        GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+
+        GridServiceDeploymentFuture old = depFuts.putIfAbsent(name, fut);
+
+        try {
+            if (old != null) {
+                if (!old.configuration().equalsIgnoreNodeFilter(cfg))
+                    throw new IgniteCheckedException("Failed to deploy service (service already exists with different " +
+                        "configuration) [deployed=" + old.configuration() + ", new=" + cfg + ']');
+                else {
+                    res.add(old, false);
+
+                    return;
                 }
+            }
+
+            GridServiceDeploymentKey key = new GridServiceDeploymentKey(name);
+
+            GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
+                new GridServiceDeployment(ctx.localNodeId(), cfg));
+
+            if (dep != null) {
+                if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
+                    String err = "Failed to deploy service (service already exists with different " +
+                        "configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']';
+
+                    U.error(log, err);
+
+                    throw new IgniteCheckedException(err);
+                }
+                else {
+                    res.add(fut, false);
+
+                    Iterator<Cache.Entry<Object, Object>> it = serviceEntries(ServiceAssignmentsPredicate.INSTANCE);
+
+                    while (it.hasNext()) {
+                        Cache.Entry<Object, Object> e = it.next();
+
+                        if (e.getKey() instanceof GridServiceAssignmentsKey) {
+                            GridServiceAssignments assigns = (GridServiceAssignments)e.getValue();
 
-                U.error(log, "Failed to deploy service: " + cfg.getName(), e);
+                            if (assigns.name().equals(name)) {
+                                fut.onDone();
 
-                return new GridFinishedFuture<>(e);
+                                depFuts.remove(name, fut);
+
+                                break;
+                            }
+                        }
+                    }
+                }
             }
+            else
+                res.add(fut, true);
         }
+        catch (IgniteCheckedException e) {
+            fut.onDone(e);
+
+            res.add(fut, false);
+
+            depFuts.remove(name, fut);
+
+            throw e;
+        }
+    }
+
+    /**
+     * @param cfg Service configuration.
+     * @return Future for deployment.
+     */
+    public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
+        A.notNull(cfg, "cfg");
+
+        return deployAll(Collections.singleton(cfg), false);
     }
 
     /**
@@ -635,37 +779,19 @@ private ServicesCompatibilityState markCompatibilityStateAsUsed() {
      * @return Future.
      */
     public IgniteInternalFuture<?> cancel(String name) {
-        ctx.security().authorize(name, SecurityPermission.SERVICE_CANCEL, null);
-
         while (true) {
             try {
-                GridFutureAdapter<?> fut = new GridFutureAdapter<>();
-
-                GridFutureAdapter<?> old;
-
-                if ((old = undepFuts.putIfAbsent(name, fut)) != null)
-                    fut = old;
-                else {
-                    GridServiceDeploymentKey key = new GridServiceDeploymentKey(name);
-
-                    if (cache.getAndRemove(key) == null) {
-                        // Remove future from local map if service was not deployed.
-                        undepFuts.remove(name);
-
-                        fut.onDone();
-                    }
-                }
-
-                return fut;
+                return removeServiceFromCache(name).fut;
             }
-            catch (ClusterTopologyCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
-            }
-            catch (IgniteCheckedException e) {
-                log.error("Failed to undeploy service: " + name, e);
+            catch (IgniteException | IgniteCheckedException e) {
+                if (X.hasCause(e, ClusterTopologyCheckedException.class)) {
+                    if (log.isDebugEnabled())
+                        log.debug("Topology changed while cancelling service (will retry): " + e.getMessage());
+                } else {
+                    U.error(log, "Failed to undeploy service: " + name, e);
 
-                return new GridFinishedFuture<>(e);
+                    return new GridFinishedFuture<>(e);
+                }
             }
         }
     }
@@ -677,7 +803,7 @@ public IgniteInternalFuture<?> cancel(String name) {
     public IgniteInternalFuture<?> cancelAll() {
         Iterator<Cache.Entry<Object, Object>> it = serviceEntries(ServiceDeploymentPredicate.INSTANCE);
 
-        GridCompoundFuture res = null;
+        List<String> svcNames = new ArrayList<>();
 
         while (it.hasNext()) {
             Cache.Entry<Object, Object> e = it.next();
@@ -687,11 +813,68 @@ public IgniteInternalFuture<?> cancelAll() {
 
             GridServiceDeployment dep = (GridServiceDeployment)e.getValue();
 
-            if (res == null)
-                res = new GridCompoundFuture<>();
+            svcNames.add(dep.configuration().getName());
+        }
+
+        return cancelAll(svcNames);
+    }
+
+    /**
+     * @param svcNames Name of service to deploy.
+     * @return Future.
+     */
+    @SuppressWarnings("unchecked")
+    public IgniteInternalFuture<?> cancelAll(Collection<String> svcNames) {
+        List<String> svcNamesCp = new ArrayList<>(svcNames);
+
+        Collections.sort(svcNamesCp);
+
+        GridCompoundFuture res;
+
+        while (true) {
+            res = null;
+
+            List<String> toRollback = new ArrayList<>();
+
+            try (Transaction tx = cache.txStart(PESSIMISTIC, READ_COMMITTED)) {
+                for (String name : svcNames) {
+                    if (res == null)
+                        res = new GridCompoundFuture<>();
+
+                    try {
+                        CancelResult cr = removeServiceFromCache(name);
+
+                        if (cr.rollback)
+                            toRollback.add(name);
 
-            // Cancel each service separately.
-            res.add(cancel(dep.configuration().getName()));
+                        res.add(cr.fut);
+                    }
+                    catch (IgniteException | IgniteCheckedException e) {
+                        if (X.hasCause(e, ClusterTopologyCheckedException.class))
+                            throw e; // Retry.
+                        else {
+                            U.error(log, "Failed to undeploy service: " + name, e);
+
+                            res.add(new GridFinishedFuture<>(e));
+                        }
+                    }
+                }
+
+                tx.commit();
+
+                break;
+            }
+            catch (IgniteException | IgniteCheckedException e) {
+                for (String name : toRollback)
+                    undepFuts.remove(name).onDone(e);
+
+                if (X.hasCause(e, ClusterTopologyCheckedException.class)) {
+                    if (log.isDebugEnabled())
+                        log.debug("Topology changed while cancelling service (will retry): " + e.getMessage());
+                }
+                else
+                    return new GridFinishedFuture<>(e);
+            }
         }
 
         if (res != null) {
@@ -703,6 +886,50 @@ public IgniteInternalFuture<?> cancelAll() {
             return new GridFinishedFuture<>();
     }
 
+    /**
+     * @param name Name of service to remove from internal cache.
+     * @return Cancellation future and a flag whether it should be completed and removed on error.
+     * @throws IgniteCheckedException If operation failed.
+     */
+    private CancelResult removeServiceFromCache(String name) throws IgniteCheckedException {
+        try {
+            ctx.security().authorize(name, SecurityPermission.SERVICE_CANCEL, null);
+        }
+        catch (SecurityException e) {
+            return new CancelResult(new GridFinishedFuture<>(e), false);
+        }
+
+        GridFutureAdapter<?> fut = new GridFutureAdapter<>();
+
+        GridFutureAdapter<?> old = undepFuts.putIfAbsent(name, fut);
+
+        if (old != null)
+            return new CancelResult(old, false);
+        else {
+            GridServiceDeploymentKey key = new GridServiceDeploymentKey(name);
+
+            try {
+                if (cache.getAndRemove(key) == null) {
+                    // Remove future from local map if service was not deployed.
+                    undepFuts.remove(name, fut);
+
+                    fut.onDone();
+
+                    return new CancelResult(fut, false);
+                }
+                else
+                    return new CancelResult(fut, true);
+            }
+            catch (IgniteCheckedException e) {
+                undepFuts.remove(name, fut);
+
+                fut.onDone(e);
+
+                throw e;
+            }
+        }
+    }
+
     /**
      * @param name Service name.
      * @param timeout If greater than 0 limits task execution time. Cannot be negative.
@@ -1854,6 +2081,26 @@ private void undeploy(String name) {
         }
     }
 
+    /**
+     *
+     */
+    private static class CancelResult {
+        /** */
+        IgniteInternalFuture<?> fut;
+
+        /** */
+        boolean rollback;
+
+        /**
+         * @param fut Future.
+         * @param rollback {@code True} if service was cancelled during current call.
+         */
+        CancelResult(IgniteInternalFuture<?> fut, boolean rollback) {
+            this.fut = fut;
+            this.rollback = rollback;
+        }
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
new file mode 100644
index 0000000000000..a581e159348a8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.services.ServiceConfiguration;
+
+/**
+ * Result of services validation before deployment.
+ */
+class PreparedConfigurations {
+    /** */
+    final List<ServiceConfiguration> cfgs;
+
+    /** */
+    final List<GridServiceDeploymentFuture> failedFuts;
+
+    /** */
+    final Exception err;
+
+    /**
+     * @param cfgs Configurations to deploy.
+     * @param failedFuts Finished futures for failed configurations.
+     * @param err Error if need to stop deploy.
+     */
+    PreparedConfigurations(List<ServiceConfiguration> cfgs, List<GridServiceDeploymentFuture> failedFuts,
+        Exception err) {
+        this.cfgs = cfgs;
+        this.failedFuts = failedFuts;
+        this.err = err;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(PreparedConfigurations.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java
new file mode 100644
index 0000000000000..32fbf6f254250
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.Collection;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Exception indicating service deployment failure.
+ */
+public class ServiceDeploymentException extends IgniteException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final Collection<ServiceConfiguration> cfgs;
+
+    /**
+     * Creates service deployment exception with error message.
+     *
+     * @param msg Error message.
+     * @param cfgs Configurations of services that failed to deploy.
+     */
+    public ServiceDeploymentException(String msg, Collection<ServiceConfiguration> cfgs) {
+        super(msg);
+
+        this.cfgs = cfgs;
+    }
+
+    /**
+     * Creates service deployment exception with {@link Throwable} as a cause.
+     *
+     * @param cause Cause.
+     * @param cfgs Configurations of services that failed to deploy.
+     */
+    public ServiceDeploymentException(Throwable cause, Collection<ServiceConfiguration> cfgs) {
+        super(cause);
+
+        this.cfgs = cfgs;
+    }
+
+    /**
+     * Creates service deployment exception with error message and {@link Throwable} as a cause.
+     *
+     * @param msg Error message.
+     * @param cause Cause.
+     * @param cfgs Configurations of services that failed to deploy.
+     */
+    public ServiceDeploymentException(String msg, @Nullable Throwable cause, Collection<ServiceConfiguration> cfgs) {
+        super(msg, cause);
+
+        this.cfgs = cfgs;
+    }
+
+    /**
+     * @return Configurations of services that failed to deploy.
+     */
+    public Collection<ServiceConfiguration> getFailedConfigurations() {
+        return cfgs;
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
index 7abd3673df9c8..0e9575fe0d70d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
@@ -113,11 +113,11 @@ public GridCompoundFuture(@Nullable IgniteReducer<T, R> rdc) {
         }
         catch (IgniteTxOptimisticCheckedException | IgniteFutureCancelledCheckedException |
             ClusterTopologyCheckedException e) {
-            if (!ignoreFailure(e))
+            if (!processFailure(e, fut))
                 onDone(e);
         }
         catch (IgniteCheckedException e) {
-            if (!ignoreFailure(e)) {
+            if (!processFailure(e, fut)) {
                 U.error(null, "Failed to execute compound future reducer: " + this, e);
 
                 onDone(e);
@@ -182,6 +182,17 @@ protected boolean ignoreFailure(Throwable err) {
         return false;
     }
 
+    /**
+     * Processes error thrown by some of the inner futures.
+     *
+     * @param err Thrown exception.
+     * @param fut Failed future.
+     * @return {@code True} if this error should be ignored.
+     */
+    protected boolean processFailure(Throwable err, IgniteInternalFuture<T> fut) {
+        return ignoreFailure(err);
+    }
+
     /**
      * Checks if there are pending futures. This is not the same as
      * {@link #isDone()} because child classes may override {@link #onDone(Object, Throwable)}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
index 723dff7a3e38f..257f1996ddd08 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
@@ -361,7 +361,7 @@ public boolean onDone(@Nullable R res, @Nullable Throwable err) {
      * @param cancel {@code True} if future is being cancelled.
      * @return {@code True} if result was set by this call.
      */
-    private boolean onDone(@Nullable R res, @Nullable Throwable err, boolean cancel) {
+    protected boolean onDone(@Nullable R res, @Nullable Throwable err, boolean cancel) {
         boolean notify = false;
 
         try {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
new file mode 100644
index 0000000000000..51c3407e65da1
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/** */
+public class GridServiceDeploymentCompoundFutureSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static GridKernalContext ctx;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        IgniteKernal kernal = (IgniteKernal)startGrid(0);
+
+        ctx = kernal.context();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWaitForCompletionOnFailingFuturePartial() throws Exception {
+        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(false, ctx);
+
+        int failingFutsNum = 2;
+
+        int completingFutsNum = 5;
+
+        Collection<GridServiceDeploymentFuture> failingFuts = new ArrayList<>(completingFutsNum);
+
+        for (int i = 0; i < failingFutsNum; i++) {
+            ServiceConfiguration failingCfg = config("Failed-" + i);
+
+            GridServiceDeploymentFuture failingFut = new GridServiceDeploymentFuture(failingCfg);
+
+            failingFuts.add(failingFut);
+
+            compFut.add(failingFut);
+        }
+
+        List<GridFutureAdapter<Object>> futs = new ArrayList<>(completingFutsNum);
+
+        for (int i = 0; i < completingFutsNum; i++) {
+            GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(config(String.valueOf(i)));
+
+            futs.add(fut);
+
+            compFut.add(fut);
+        }
+
+        compFut.serviceDeploymentMarkInitialized();
+
+        List<Exception> causes = new ArrayList<>();
+
+        for (GridServiceDeploymentFuture fut : failingFuts) {
+            Exception cause = new Exception("Test error");
+
+            causes.add(cause);
+
+            fut.onDone(cause);
+        }
+
+        try {
+            compFut.get(100);
+
+            fail("Should never reach here.");
+        }
+        catch (IgniteFutureTimeoutCheckedException e) {
+            log.info("Expected exception: " + e.getMessage());
+        }
+
+        for (GridFutureAdapter<Object> fut : futs)
+            fut.onDone();
+
+        try {
+            compFut.get();
+
+            fail("Should never reach here.");
+        }
+        catch (IgniteCheckedException ce) {
+            log.info("Expected exception: " + ce.getMessage());
+
+            IgniteException e = U.convertException(ce);
+
+            assertTrue(e instanceof ServiceDeploymentException);
+
+            Throwable[] supErrs = e.getSuppressed();
+
+            assertEquals(failingFutsNum, supErrs.length);
+
+            for (int i = 0; i < failingFutsNum; i++)
+                assertEquals(causes.get(i), supErrs[i].getCause());
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testFailAllAfterInitialized() throws Exception {
+        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(true, ctx);
+
+        ServiceConfiguration failingCfg = config("Failed");
+
+        GridServiceDeploymentFuture failingFut = new GridServiceDeploymentFuture(failingCfg);
+
+        compFut.add(failingFut);
+
+        int futsNum = 5;
+
+        List<ServiceConfiguration> cfgs = new ArrayList<>(futsNum + 1);
+
+        cfgs.add(failingCfg);
+
+        for (int i = 0; i < futsNum; i++) {
+            ServiceConfiguration cfg = config(String.valueOf(i));
+
+            cfgs.add(cfg);
+
+            compFut.add(new GridServiceDeploymentFuture(cfg));
+        }
+
+        compFut.serviceDeploymentMarkInitialized();
+
+        Exception expCause = new Exception("Test error");
+
+        failingFut.onDone(expCause);
+
+        assertFailAll(compFut, cfgs, expCause);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testFailAllBeforeInitialized() throws Exception {
+        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(true, ctx);
+
+        ServiceConfiguration failingCfg = config("Failed");
+
+        GridServiceDeploymentFuture failingFut = new GridServiceDeploymentFuture(failingCfg);
+
+        Exception expCause = new Exception("Test error");
+
+        failingFut.onDone(expCause);
+
+        compFut.add(failingFut);
+
+        assertFalse(compFut.isDone());
+
+        int futsNum = 5;
+
+        List<ServiceConfiguration> cfgs = new ArrayList<>(futsNum + 1);
+
+        cfgs.add(failingCfg);
+
+        for (int i = 0; i < futsNum; i++) {
+            ServiceConfiguration cfg = config(String.valueOf(i));
+
+            cfgs.add(cfg);
+
+            compFut.add(new GridServiceDeploymentFuture(cfg));
+        }
+
+        compFut.serviceDeploymentMarkInitialized();
+
+        assertFailAll(compFut, cfgs, expCause);
+    }
+
+    /**
+     * Try waiting for the future completion and check that a proper exception is thrown.
+     *
+     * @param fut Future.
+     * @param expCfgs Expected cfgs.
+     * @param expCause Expected cause.
+     */
+    private void assertFailAll(GridServiceDeploymentCompoundFuture fut, Collection<ServiceConfiguration> expCfgs,
+        Exception expCause) {
+        try {
+            fut.get();
+
+            fail("Should never reach here.");
+        }
+        catch (IgniteCheckedException ce) {
+            log.info("Expected exception: " + ce.getMessage());
+
+            IgniteException e = U.convertException(ce);
+
+            assertTrue(e instanceof ServiceDeploymentException);
+
+            assertEqualsCollections(expCfgs, ((ServiceDeploymentException)e).getFailedConfigurations());
+
+            Throwable actCause = e.getCause();
+
+            assertTrue(actCause instanceof IgniteCheckedException);
+
+            assertEquals(expCause, actCause.getCause());
+        }
+    }
+
+    /**
+     * @param name Name.
+     * @return Dummy configuration with a specified name.
+     */
+    private ServiceConfiguration config(String name) {
+        ServiceConfiguration cfg = new ServiceConfiguration();
+
+        cfg.setName(name);
+
+        return cfg;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
new file mode 100644
index 0000000000000..c3232f23e9c2f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
@@ -0,0 +1,765 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteServices;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceDescriptor;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+
+/**
+ * Test for deployment of multiple configurations at a time.
+ */
+public class GridServiceProcessorBatchDeploySelfTest extends GridCommonAbstractTest {
+    /** Number of services to be deployed. */
+    private static final int NUM_SERVICES = 100;
+
+    /** Number of nodes in the test cluster. */
+    private static final int NUM_NODES = 4;
+
+    /** Client node name. */
+    private static final String CLIENT_NODE_NAME = "client";
+
+    /** IP finder. */
+    private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+        discoSpi.setIpFinder(ipFinder);
+
+        c.setDiscoverySpi(discoSpi);
+
+        return c;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        for (int i = 0; i < NUM_NODES; i++)
+            startGrid(i);
+
+        startGrid(CLIENT_NODE_NAME, getConfiguration(CLIENT_NODE_NAME).setClientMode(true));
+
+        DummyService.reset();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAll() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(cfgs, false);
+
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+        assertDeployedServices(client, cfgs);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAllAsync() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        IgniteServices services =  client.services().withAsync();
+
+        services.deployAll(cfgs, false);
+
+        services.future().get();
+
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+        assertDeployedServices(client, cfgs);
+    }
+
+    /**
+     * TODO: enable when IGNITE-6259 is fixed
+     *
+     * @throws Exception If failed.
+     */
+    public void _testDeployAllTopologyChange() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        final AtomicBoolean finished = new AtomicBoolean();
+
+        IgniteInternalFuture<Object> topChangeFut = runTopChanger(finished);
+
+        try {
+            int numServices = 500;
+            int batchSize = 5;
+
+            CountDownLatch latch = new CountDownLatch(numServices);
+
+            IgnitePredicate<ClusterNode> depPred = client.cluster().forServers()
+                .forPredicate(new IgnitePredicate<ClusterNode>() {
+                    @Override public boolean apply(ClusterNode node) {
+                        String gridName = node.attribute(IgniteNodeAttributes.ATTR_GRID_NAME);
+
+                        assert gridName != null;
+
+                        return gridName.startsWith(getTestGridName());
+                    }
+                }).predicate();
+
+            List<ServiceConfiguration> cfgs = getConfigs(depPred, numServices);
+
+            subscribeExeLatch(cfgs, latch);
+
+            IgniteServices services = client.services().withAsync();
+
+            int from = 0;
+
+            while (from < numServices) {
+                int to = Math.min(numServices, from + batchSize);
+
+                services.deployAll(cfgs.subList(from, to), false);
+
+                services.future().get(5000);
+
+                from = to;
+            }
+
+            assertTrue(latch.await(30, TimeUnit.SECONDS));
+
+            assertDeployedServices(client, cfgs);
+        }
+        finally {
+            finished.set(true);
+        }
+
+        topChangeFut.get();
+    }
+
+    /**
+     * TODO: enable when IGNITE-6259 is fixed
+     *
+     * @throws Exception If failed.
+     */
+    public void _testDeployAllTopologyChangeFail() throws Exception {
+        final Ignite client = grid(CLIENT_NODE_NAME);
+
+        final AtomicBoolean finished = new AtomicBoolean();
+
+        IgniteInternalFuture<Object> topChangeFut = runTopChanger(finished);
+
+        try {
+            int numServices = 500;
+            int batchSize = 5;
+
+            CountDownLatch latch = new CountDownLatch(numServices);
+
+            IgnitePredicate<ClusterNode> depPred = client.cluster().forServers()
+                .forPredicate(new IgnitePredicate<ClusterNode>() {
+                    @Override public boolean apply(ClusterNode node) {
+                        String gridName = node.attribute(IgniteNodeAttributes.ATTR_GRID_NAME);
+
+                        assert gridName != null;
+
+                        return gridName.startsWith(getTestGridName());
+                    }
+                }).predicate();
+
+            List<ServiceConfiguration> cfgs = getConfigs(depPred, numServices);
+
+            List<ServiceConfiguration> failingCfgs = new ArrayList<>();
+
+            subscribeExeLatch(cfgs, latch);
+
+            IgniteServices services = client.services().withAsync();
+
+            int from = 0;
+
+            while (from < numServices) {
+                int to = Math.min(numServices, from + batchSize);
+
+                List<ServiceConfiguration> cfgsBatch = cfgs.subList(from, to);
+
+                ServiceConfiguration failingCfg = cfgsBatch.get(0);
+
+                failingCfg.setName(null);
+
+                failingCfgs.add(failingCfg);
+
+                try {
+                    services.deployAll(cfgsBatch, false);
+
+                    services.future().get(5000);
+
+                    fail("Should never reach here.");
+                }
+                catch (ServiceDeploymentException e) {
+                    assertEquals(1, e.getFailedConfigurations().size());
+
+                    ServiceConfiguration actFailedCfg = copyService(e.getFailedConfigurations().iterator().next());
+
+                    assertEquals(failingCfg, actFailedCfg);
+
+                    latch.countDown();
+                }
+
+                from = to;
+            }
+
+            assertTrue(latch.await(30, TimeUnit.SECONDS));
+
+            cfgs.removeAll(failingCfgs);
+
+            assertDeployedServices(client, cfgs);
+        }
+        finally {
+            finished.set(true);
+        }
+
+        topChangeFut.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAllFailAll() throws Exception {
+        deployAllFail(false, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAllPartial() throws Exception {
+        deployAllFail(false, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAllAsyncFailAll() throws Exception {
+        deployAllFail(true, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployAllAsyncFailPartial() throws Exception {
+        deployAllFail(true, false);
+    }
+
+    /**
+     * @param async If {@code true}, then asynchronous method of deployment will be performed.
+     * @param allOrNone Failing strategy.
+     * @throws Exception If failed.
+     */
+    private void deployAllFail(boolean async, boolean allOrNone) throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES - 1);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        ServiceConfiguration failingCfg = cfgs.get(cfgs.size() - 1);
+
+        failingCfg.setName(null);
+
+        assertFailingDeploy(client, async, allOrNone, cfgs, failingCfg);
+
+        if (allOrNone) {
+            assertFalse("Some of the services were deployed.", latch.await(2, TimeUnit.SECONDS));
+
+            assertEquals(NUM_SERVICES - 1, latch.getCount());
+
+            assertTrue(client.services().serviceDescriptors().isEmpty());
+        }
+        else {
+            assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+            assertDeployedServices(client, cfgs.subList(0, cfgs.size() - 1));
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClashingNames() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        List<ServiceConfiguration> fstBatch = cfgs.subList(0, NUM_SERVICES / 2);
+        List<ServiceConfiguration> sndBatch = cfgs.subList(NUM_SERVICES / 4, NUM_SERVICES);
+
+        IgniteServices svcs1 = client.services().withAsync();
+        IgniteServices svcs2 = client.services().withAsync();
+
+        svcs1.deployAll(fstBatch, false);
+        svcs2.deployAll(sndBatch, false);
+
+        svcs1.future().get();
+        svcs2.future().get();
+
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+        assertDeployedServices(client, cfgs);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClashingNamesFailAll() throws Exception {
+        clashingNamesFail(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClashingNamesPartial() throws Exception {
+        clashingNamesFail(false);
+    }
+
+    /**
+     * @param allOrNone Failing strategy.
+     * @throws Exception If failed.
+     */
+    private void clashingNamesFail(boolean allOrNone) throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        int numDepSvcs;
+
+        if (allOrNone)
+            numDepSvcs = NUM_SERVICES / 2;
+        else
+            numDepSvcs = NUM_SERVICES - 1;
+
+        CountDownLatch latch = new CountDownLatch(numDepSvcs);
+
+        List<ServiceConfiguration> fstBatch = cfgs.subList(0, NUM_SERVICES / 2);
+        List<ServiceConfiguration> sndBatch = cfgs.subList(NUM_SERVICES / 4, NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        IgniteServices services = client.services().withAsync();
+
+        services.deployAll(fstBatch, false);
+
+        ServiceConfiguration failingCfg = cfgs.get(NUM_SERVICES - 1);
+
+        failingCfg.setName(null);
+
+        assertFailingDeploy(client, false, allOrNone, sndBatch, failingCfg);
+
+        services.future().get();
+
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+        assertDeployedServices(client, cfgs.subList(0, numDepSvcs));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClashingNameDifferentConfigFailAll() throws Exception {
+        testClashingNameDifferentConfig(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClashingNameDifferentConfigPartial() throws Exception {
+        testClashingNameDifferentConfig(false);
+    }
+
+    /**
+     * @param allOrNone Failing strategy.
+     * @throws Exception If failed.
+     */
+    private void testClashingNameDifferentConfig(boolean allOrNone) throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        int numDepSvcs;
+
+        if (allOrNone)
+            numDepSvcs = NUM_SERVICES / 2;
+        else
+            numDepSvcs = NUM_SERVICES - 1;
+
+
+        CountDownLatch latch = new CountDownLatch(numDepSvcs);
+
+        List<ServiceConfiguration> fstBatch = cfgs.subList(0, NUM_SERVICES / 2);
+        List<ServiceConfiguration> sndBatch = cfgs.subList(NUM_SERVICES / 4, NUM_SERVICES - 1);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(fstBatch, false);
+
+        ServiceConfiguration failingCfg = copyService(cfgs.get(NUM_SERVICES - 1));
+
+        // Same name, different config.
+        failingCfg.setName(fstBatch.get(0).getName());
+        failingCfg.setTotalCount(fstBatch.get(0).getTotalCount() + 1);
+
+        sndBatch.add(failingCfg);
+
+        assertFailingDeploy(client, false, allOrNone, sndBatch, failingCfg);
+
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+
+        assertDeployedServices(client, cfgs.subList(0, numDepSvcs));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCancelAll() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(cfgs, true);
+
+        latch.await(30, TimeUnit.SECONDS);
+
+        client.services().cancelAll();
+
+        assertDeployedServices(client, Collections.<ServiceConfiguration>emptyList());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCancelAllAsync() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(cfgs, true);
+
+        latch.await(30, TimeUnit.SECONDS);
+
+        IgniteServices services = client.services().withAsync();
+
+        services.cancelAll();
+
+        services.future().get();
+
+        assertDeployedServices(client, Collections.<ServiceConfiguration>emptyList());
+    }
+
+    /**
+     * TODO: enable when IGNITE-6259 is fixed
+     *
+     * @throws Exception If failed.
+     */
+    public void _testCancelAllTopologyChange() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        int numServices = 500;
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), numServices);
+
+        CountDownLatch latch = new CountDownLatch(numServices);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(cfgs, true);
+
+        latch.await(30, TimeUnit.SECONDS);
+
+        final AtomicBoolean finished = new AtomicBoolean();
+
+        IgniteInternalFuture<Object> topChangeFut = runTopChanger(finished);
+
+        List<String> names = new ArrayList<>();
+
+        for (ServiceConfiguration cfg : cfgs)
+            names.add(cfg.getName());
+
+        try {
+            IgniteServices services = client.services().withAsync();
+
+            int batchSize = 5;
+            int from = 0;
+
+            while (from < numServices) {
+                int to = Math.min(numServices, from + batchSize);
+
+                log.info("Trying to cancel services [" + from + ".." + to + ")");
+
+                services.cancelAll(names.subList(from, to));
+
+                services.future().get(5000);
+
+                from = to;
+            }
+
+            assertDeployedServices(client, Collections.<ServiceConfiguration>emptyList());
+        }
+        finally {
+            finished.set(true);
+        }
+
+        topChangeFut.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCancelAllClashingNames() throws Exception {
+        Ignite client = grid(CLIENT_NODE_NAME);
+
+        List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
+
+        CountDownLatch latch = new CountDownLatch(NUM_SERVICES);
+
+        subscribeExeLatch(cfgs, latch);
+
+        client.services().deployAll(cfgs, true);
+
+        latch.await(30, TimeUnit.SECONDS);
+
+        List<String> names = new ArrayList<>();
+
+        for (ServiceConfiguration cfg : cfgs)
+            names.add(cfg.getName());
+
+        int batchSize = 5;
+        int from = 0;
+
+        while (from < NUM_SERVICES) {
+            int to = Math.min(NUM_SERVICES, from + batchSize);
+
+            List<String> toCancel = new ArrayList<>(names.subList(from, to));
+
+            toCancel.add(toCancel.get(0));
+
+            client.services().cancelAll(toCancel);
+
+            from = to;
+        }
+
+        assertDeployedServices(client, Collections.<ServiceConfiguration>emptyList());
+    }
+
+    /**
+     * @param client Client.
+     * @param async If {@code true}, then async version of deploy method will be used.
+     * @param cfgs Service configurations.
+     * @param allOrNone Failing policy.
+     * @param failingCfg Configuration of the failing service.
+     * @throws Exception If failed.
+     */
+    private void assertFailingDeploy(Ignite client, boolean async, boolean allOrNone, List<ServiceConfiguration> cfgs,
+        ServiceConfiguration failingCfg) throws Exception {
+
+        IgniteServices services = client.services();
+
+        if (async) {
+            services = services.withAsync();
+
+            services.deployAll(cfgs, allOrNone);
+        }
+
+        try {
+            if (async)
+                services.future().get();
+            else
+                services.deployAll(cfgs, allOrNone);
+
+            fail("Should never reach here.");
+        }
+        catch (ServiceDeploymentException e) {
+            info("Expected exception: " + e.getMessage());
+
+            Collection<ServiceConfiguration> expFails;
+
+            if (allOrNone)
+                expFails = cfgs;
+            else
+                expFails = Collections.singleton(failingCfg);
+
+            Collection<ServiceConfiguration> actFails = e.getFailedConfigurations();
+
+            // Some cfgs may be lazy. Construct ServiceConfiguration from them for comparison.
+            Collection<ServiceConfiguration> actFailsCp = new ArrayList<>(actFails.size());
+
+            for (ServiceConfiguration cfg : actFails)
+                actFailsCp.add(copyService(cfg));
+
+            assertEqualsCollections(expFails, actFailsCp);
+        }
+    }
+
+    /**
+     * @param cfg Config.
+     * @return Copy of provided configuration.
+     */
+    private ServiceConfiguration copyService(ServiceConfiguration cfg) {
+        ServiceConfiguration cfgCp = new ServiceConfiguration();
+
+        cfgCp.setName(cfg.getName());
+
+        cfgCp.setMaxPerNodeCount(cfg.getMaxPerNodeCount());
+
+        cfgCp.setTotalCount(cfg.getTotalCount());
+
+        cfgCp.setAffinityKey(cfg.getAffinityKey());
+
+        cfgCp.setCacheName(cfg.getCacheName());
+
+        cfgCp.setName(cfg.getName());
+
+        cfgCp.setService(cfg.getService());
+
+        cfgCp.setNodeFilter(cfg.getNodeFilter());
+
+        return cfgCp;
+    }
+
+    /**
+     * @param client Client Ignite instance.
+     * @param expCfgs Configurations of services that are expected to be deployed.
+     */
+    private void assertDeployedServices(Ignite client, Collection<ServiceConfiguration> expCfgs) {
+        Set<String> expNames = new HashSet<>();
+        Set<String> actNames = new HashSet<>();
+
+        for (ServiceConfiguration cfg : expCfgs)
+            expNames.add(cfg.getName());
+
+        for (ServiceDescriptor desc : client.services().serviceDescriptors())
+            actNames.add(desc.name());
+
+        assertEquals(expNames, actNames);
+    }
+
+    /**
+     * @param nodePred Node predicate.
+     * @param numServices Number of configurations to generate.
+     * @return Generated services configurations.
+     */
+    private List<ServiceConfiguration> getConfigs(IgnitePredicate<ClusterNode> nodePred, int numServices) {
+        List<ServiceConfiguration> cfgs = new ArrayList<>(numServices);
+
+        for (int i = 0; i < numServices; i++) {
+            String name = "testService-" + i;
+
+            ServiceConfiguration cfg = new ServiceConfiguration();
+
+            cfg.setName(name);
+            cfg.setTotalCount(1);
+            cfg.setMaxPerNodeCount(1);
+            cfg.setService(new DummyService());
+            cfg.setNodeFilter(nodePred);
+
+            cfgs.add(cfg);
+        }
+        return cfgs;
+    }
+
+    /**
+     * @param cfgs Configurations.
+     * @param latch Latch.
+     */
+    private void subscribeExeLatch(List<ServiceConfiguration> cfgs, CountDownLatch latch) {
+        for (ServiceConfiguration cfg : cfgs)
+            DummyService.exeLatch(cfg.getName(), latch);
+    }
+
+    /**
+     * @param finished Finished flag.
+     * @throws Exception If failed.
+     * @return Future.
+     */
+    private IgniteInternalFuture<Object> runTopChanger(final AtomicBoolean finished) throws Exception {
+        return runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                String namePrefix = "extra-node-";
+
+                int extraNodesNum = 3;
+
+                while (!finished.get()) {
+                    for (int i = 0; i < extraNodesNum; i++)
+                        startGrid(namePrefix + i);
+
+                    for (int i = 0; i < extraNodesNum; i++)
+                        stopGrid(namePrefix + i);
+
+                    awaitPartitionMapExchange();
+                }
+
+                return null;
+            }
+        });
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 43bd3a44c6388..8ff393c9eea32 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -50,7 +50,9 @@
 import org.apache.ignite.internal.processors.port.GridPortProcessorSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceClientNodeTest;
 import org.apache.ignite.internal.processors.service.GridServiceContinuousQueryRedeploy;
+import org.apache.ignite.internal.processors.service.GridServiceDeploymentCompoundFutureSelfTest;
 import org.apache.ignite.internal.processors.service.GridServicePackagePrivateSelfTest;
+import org.apache.ignite.internal.processors.service.GridServiceProcessorBatchDeploySelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceProcessorMultiNodeConfigSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceProcessorMultiNodeSelfTest;
 import org.apache.ignite.internal.processors.service.GridServiceProcessorProxySelfTest;
@@ -146,6 +148,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(IgniteServiceDynamicCachesSelfTest.class);
         suite.addTestSuite(GridServiceContinuousQueryRedeploy.class);
         suite.addTestSuite(ServiceThreadPoolSelfTest.class);
+        suite.addTestSuite(GridServiceProcessorBatchDeploySelfTest.class);
+        suite.addTestSuite(GridServiceDeploymentCompoundFutureSelfTest.class);
 
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class);

From ec4cd64a894f85f20d4ff8d24b143a0da61d1fc5 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 5 Sep 2017 17:03:57 +0300
Subject: [PATCH 360/516] ignite-6273 Moved ServiceDeploymentException to
 org.apache.ignite.services

(cherry picked from commit d7e215d)
---
 .../core/src/main/java/org/apache/ignite/IgniteServices.java   | 2 +-
 .../service/GridServiceDeploymentCompoundFuture.java           | 1 +
 .../internal/processors/service/GridServiceProcessor.java      | 1 +
 .../service => services}/ServiceDeploymentException.java       | 3 +--
 .../service/GridServiceDeploymentCompoundFutureSelfTest.java   | 1 +
 .../service/GridServiceProcessorBatchDeploySelfTest.java       | 1 +
 6 files changed, 6 insertions(+), 3 deletions(-)
 rename modules/core/src/main/java/org/apache/ignite/{internal/processors/service => services}/ServiceDeploymentException.java (95%)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
index ae3958fc269cf..369baebfb6237 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
@@ -20,7 +20,7 @@
 import java.util.Collection;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.processors.service.ServiceDeploymentException;
+import org.apache.ignite.services.ServiceDeploymentException;
 import org.apache.ignite.lang.IgniteAsyncSupport;
 import org.apache.ignite.lang.IgniteAsyncSupported;
 import org.apache.ignite.resources.IgniteInstanceResource;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
index 12b88e5f119f5..bbf03702a226f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceDeploymentException;
 import org.jetbrains.annotations.Nullable;
 
 /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index f457d8091a4ac..2a30bad8d4950 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -100,6 +100,7 @@
 import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceDeploymentException;
 import org.apache.ignite.services.ServiceDescriptor;
 import org.apache.ignite.spi.IgniteNodeValidationResult;
 import org.apache.ignite.thread.IgniteThreadFactory;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java b/modules/core/src/main/java/org/apache/ignite/services/ServiceDeploymentException.java
similarity index 95%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java
rename to modules/core/src/main/java/org/apache/ignite/services/ServiceDeploymentException.java
index 32fbf6f254250..843a8fe37c59a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentException.java
+++ b/modules/core/src/main/java/org/apache/ignite/services/ServiceDeploymentException.java
@@ -15,11 +15,10 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.service;
+package org.apache.ignite.services;
 
 import java.util.Collection;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.services.ServiceConfiguration;
 import org.jetbrains.annotations.Nullable;
 
 /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
index 51c3407e65da1..ca95198cf2fc9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceDeploymentException;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /** */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
index c3232f23e9c2f..337e69565968b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceDeploymentException;
 import org.apache.ignite.services.ServiceDescriptor;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;

From 1831f41d80497012b9bfff38d62099734a3af245 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Wed, 6 Sep 2017 13:27:25 +0300
Subject: [PATCH 361/516] IGNITE-6271 .NET: Propagate
 ServiceDeploymentException

(cherry picked from commit 56860d2)
---
 .../services/PlatformAbstractService.java     |   8 ++
 .../platform/services/PlatformServices.java   |  63 ++++++++--
 .../platform/utils/PlatformUtils.java         |   2 +-
 .../Services/ServicesAsyncWrapper.cs          |   9 +-
 .../Services/ServicesTest.cs                  |  80 +++++++++++--
 .../Apache.Ignite.Core.csproj                 |   1 +
 .../Apache.Ignite.Core/Impl/ExceptionUtils.cs |   6 +-
 .../Impl/Services/ServiceProxySerializer.cs   |  36 ++++++
 .../Impl/Services/Services.cs                 |  29 ++++-
 .../Impl/Unmanaged/UnmanagedCallbacks.cs      |  37 +++++-
 .../Services/ServiceDeploymentException.cs    | 111 ++++++++++++++++++
 .../dotnet/Apache.Ignite.sln.DotSettings      |   1 +
 12 files changed, 350 insertions(+), 33 deletions(-)
 create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Services/ServiceDeploymentException.cs

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformAbstractService.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformAbstractService.java
index d6a6e16f0f2c0..fc0fa14ebc5dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformAbstractService.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformAbstractService.java
@@ -94,6 +94,14 @@ public PlatformAbstractService(Object svc, PlatformContext ctx, boolean srvKeepB
             out.synchronize();
 
             ptr = platformCtx.gateway().serviceInit(mem.pointer());
+
+            PlatformInputStream in = mem.input();
+
+            in.synchronize();
+
+            BinaryRawReaderEx reader = platformCtx.reader(in);
+
+            PlatformUtils.readInvocationResult(platformCtx, reader);
         }
         catch (IgniteCheckedException e) {
             throw U.convertException(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
index c266986eb6e76..4259220752647 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.processors.platform.PlatformContext;
 import org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetService;
 import org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetServiceImpl;
+import org.apache.ignite.internal.processors.platform.utils.PlatformFutureUtils;
 import org.apache.ignite.internal.processors.platform.utils.PlatformUtils;
 import org.apache.ignite.internal.processors.platform.utils.PlatformWriterBiClosure;
 import org.apache.ignite.internal.processors.platform.utils.PlatformWriterClosure;
@@ -37,6 +38,7 @@
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.services.ServiceDescriptor;
+import org.jetbrains.annotations.NotNull;
 
 import java.lang.reflect.Method;
 import java.util.ArrayList;
@@ -103,6 +105,9 @@ public class PlatformServices extends PlatformAbstractTarget {
     private static final CopyOnWriteConcurrentMap<T3<Class, String, Integer>, Method> SVC_METHODS
         = new CopyOnWriteConcurrentMap<>();
 
+    /** Future result writer. */
+    private static final PlatformFutureUtils.Writer RESULT_WRITER = new ServiceDeploymentResultWriter();
+
     /** */
     private final IgniteServices services;
 
@@ -147,20 +152,10 @@ private ServiceDescriptor findDescriptor(String name) {
     @Override protected long processInStreamOutLong(int type, BinaryRawReaderEx reader)
         throws IgniteCheckedException {
         switch (type) {
-            case OP_DOTNET_DEPLOY: {
-                dotnetDeploy(reader, services);
-
-                return TRUE;
-            }
-
             case OP_DOTNET_DEPLOY_ASYNC: {
                 dotnetDeploy(reader, servicesAsync);
 
-                return readAndListenFuture(reader);
-            }
-
-            case OP_DOTNET_DEPLOY_MULTIPLE: {
-                dotnetDeployMultiple(reader, services);
+                readAndListenFuture(reader, currentFuture(), RESULT_WRITER);
 
                 return TRUE;
             }
@@ -168,7 +163,9 @@ private ServiceDescriptor findDescriptor(String name) {
             case OP_DOTNET_DEPLOY_MULTIPLE_ASYNC: {
                 dotnetDeployMultiple(reader, servicesAsync);
 
-                return readAndListenFuture(reader);
+                readAndListenFuture(reader, currentFuture(), RESULT_WRITER);
+
+                return TRUE;
             }
 
             case OP_CANCEL: {
@@ -217,6 +214,32 @@ private ServiceDescriptor findDescriptor(String name) {
                 return;
             }
 
+            case OP_DOTNET_DEPLOY: {
+                try {
+                    dotnetDeploy(reader, services);
+
+                    PlatformUtils.writeInvocationResult(writer, null, null);
+                }
+                catch (Exception e) {
+                    PlatformUtils.writeInvocationResult(writer, null, e);
+                }
+
+                return;
+            }
+
+            case OP_DOTNET_DEPLOY_MULTIPLE: {
+                try {
+                    dotnetDeployMultiple(reader, services);
+
+                    PlatformUtils.writeInvocationResult(writer, null, null);
+                }
+                catch (Exception e) {
+                    PlatformUtils.writeInvocationResult(writer, null, e);
+                }
+
+                return;
+            }
+
             default:
                 super.processInStreamOutStream(type, reader, writer);
         }
@@ -579,4 +602,20 @@ public void put(K key, V val) {
             }
         }
     }
+
+    /**
+     * Writes an EventBase.
+     */
+    private static class ServiceDeploymentResultWriter implements PlatformFutureUtils.Writer {
+        /** <inheritDoc /> */
+        @Override public void write(BinaryRawWriterEx writer, Object obj, Throwable err) {
+            PlatformUtils.writeInvocationResult(writer, obj, err);
+        }
+
+        /** <inheritDoc /> */
+        @Override public boolean canWrite(Object obj, Throwable err) {
+            return true;
+        }
+    }
+
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
index 959ff68a4e19f..b1f3cb0ef8d0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
@@ -765,7 +765,7 @@ public static byte[] errorData(Throwable err) {
      * @param resObj Result.
      * @param err Error.
      */
-    public static void writeInvocationResult(BinaryRawWriterEx writer, Object resObj, Exception err)
+    public static void writeInvocationResult(BinaryRawWriterEx writer, Object resObj, Throwable err)
     {
         if (err == null) {
             writer.writeBoolean(true);
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesAsyncWrapper.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesAsyncWrapper.cs
index f0740e04761cf..470804c018292 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesAsyncWrapper.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesAsyncWrapper.cs
@@ -104,7 +104,14 @@ public Task DeployMultipleAsync(string name, IService service, int totalCount, i
         /** <inheritDoc /> */
         public void Deploy(ServiceConfiguration configuration)
         {
-            _services.DeployAsync(configuration).Wait();
+            try
+            {
+                _services.DeployAsync(configuration).Wait();
+            }
+            catch (AggregateException ex)
+            {
+                throw ex.InnerException;
+            }
         }
 
         /** <inheritDoc /> */
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesTest.cs
index 0558d1120496e..26650863c0cd8 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Services/ServicesTest.cs
@@ -431,17 +431,71 @@ public void TestWithKeepBinaryBoth()
         /// Tests exception in Initialize.
         /// </summary>
         [Test]
-        public void TestInitException()
+        public void TestDeployMultipleException([Values(true, false)] bool keepBinary)
+        {
+            VerifyDeploymentException((services, svc) =>
+                services.DeployMultiple(SvcName, svc, Grids.Length, 1), keepBinary);
+        }
+
+        /// <summary>
+        /// Tests exception in Initialize.
+        /// </summary>
+        [Test]
+        public void TestDeployException([Values(true, false)] bool keepBinary)
+        {
+            VerifyDeploymentException((services, svc) =>
+                services.Deploy(new ServiceConfiguration
+                {
+                    Name = SvcName,
+                    Service = svc,
+                    TotalCount = Grids.Length,
+                    MaxPerNodeCount = 1
+                }), keepBinary);
+        }
+
+        /// <summary>
+        /// Verifies the deployment exception.
+        /// </summary>
+        private void VerifyDeploymentException(Action<IServices, IService> deploy, bool keepBinary)
         {
             var svc = new TestIgniteServiceSerializable { ThrowInit = true };
 
-            var ex = Assert.Throws<IgniteException>(() => Services.DeployMultiple(SvcName, svc, Grids.Length, 1));
+            var services = Services;
+
+            if (keepBinary)
+            {
+                services = services.WithKeepBinary();
+            }
+
+            var deploymentException = Assert.Throws<ServiceDeploymentException>(() => deploy(services, svc));
+
+            var text = keepBinary 
+                ? "Service deployment failed with a binary error. Examine BinaryCause for details."
+                : "Service deployment failed with an exception. Examine InnerException for details.";
+
+            Assert.AreEqual(text, deploymentException.Message);
+
+            Exception ex;
+
+            if (keepBinary)
+            {
+                Assert.IsNull(deploymentException.InnerException);
+
+                ex = deploymentException.BinaryCause.Deserialize<Exception>();
+            }
+            else
+            {
+                Assert.IsNull(deploymentException.BinaryCause);
+
+                ex = deploymentException.InnerException;
+            }
+
+            Assert.IsNotNull(ex);
             Assert.AreEqual("Expected exception", ex.Message);
-            Assert.IsNotNull(ex.InnerException);
-            Assert.IsTrue(ex.InnerException.Message.Contains("PlatformCallbackUtils.serviceInit(Native Method)"));
+            Assert.IsTrue(ex.StackTrace.Trim().StartsWith(
+                "at Apache.Ignite.Core.Tests.Services.ServicesTest.TestIgniteServiceSerializable.Init"));
 
             var svc0 = Services.GetService<TestIgniteServiceSerializable>(SvcName);
-
             Assert.IsNull(svc0);
         }
 
@@ -480,19 +534,27 @@ public void TestCancelException()
             AssertNoService();
         }
 
+        /// <summary>
+        /// Tests exception in binarizable implementation.
+        /// </summary>
         [Test]
         public void TestMarshalExceptionOnRead()
         {
             var svc = new TestIgniteServiceBinarizableErr();
 
-            var ex = Assert.Throws<IgniteException>(() => Services.DeployMultiple(SvcName, svc, Grids.Length, 1));
-            Assert.AreEqual("Expected exception", ex.Message);
+            var ex = Assert.Throws<ServiceDeploymentException>(() =>
+                Services.DeployMultiple(SvcName, svc, Grids.Length, 1));
+            
+            Assert.AreEqual("Expected exception", ex.InnerException.Message);
 
             var svc0 = Services.GetService<TestIgniteServiceSerializable>(SvcName);
 
             Assert.IsNull(svc0);
         }
 
+        /// <summary>
+        /// Tests exception in binarizable implementation.
+        /// </summary>
         [Test]
         public void TestMarshalExceptionOnWrite()
         {
@@ -506,6 +568,9 @@ public void TestMarshalExceptionOnWrite()
             Assert.IsNull(svc0);
         }
 
+        /// <summary>
+        /// Tests Java service invocation.
+        /// </summary>
         [Test]
         public void TestCallJavaService()
         {
@@ -777,6 +842,7 @@ private class TestIgniteServiceSerializable : IService, ITestIgniteService
             /** <inheritdoc /> */
             public Guid NodeId
             {
+                // ReSharper disable once InconsistentlySynchronizedField
                 get { return _grid.GetCluster().GetLocalNode().Id; }
             }
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
index f945efea27d3c..4f9d41e675b08 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
@@ -473,6 +473,7 @@
     <Compile Include="Services\IServices.cs" />
     <Compile Include="Services\Package-Info.cs" />
     <Compile Include="Services\ServiceConfiguration.cs" />
+    <Compile Include="Services\ServiceDeploymentException.cs" />
     <Compile Include="Services\ServiceInvocationException.cs" />
     <Compile Include="SwapSpace\File\FileSwapSpaceSpi.cs" />
     <Compile Include="SwapSpace\ISwapSpaceSpi.cs" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs
index ddbdd86fc559b..e4b2b062e7883 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs
@@ -30,6 +30,7 @@ namespace Apache.Ignite.Core.Impl
     using Apache.Ignite.Core.Common;
     using Apache.Ignite.Core.Compute;
     using Apache.Ignite.Core.Impl.Binary;
+    using Apache.Ignite.Core.Services;
     using Apache.Ignite.Core.Transactions;
 
     /// <summary>
@@ -103,9 +104,12 @@ static ExceptionUtils()
             Exs["org.apache.ignite.IgniteAuthenticationException"] = (i, m, e) => new SecurityException(m, e);
             Exs["org.apache.ignite.plugin.security.GridSecurityException"] = (i, m, e) => new SecurityException(m, e);
 
-            // Future exceptions
+            // Future exceptions.
             Exs["org.apache.ignite.lang.IgniteFutureCancelledException"] = (i, m, e) => new IgniteFutureCancelledException(m, e);
             Exs["org.apache.ignite.internal.IgniteFutureCancelledCheckedException"] = (i, m, e) => new IgniteFutureCancelledException(m, e);
+
+            // Service exceptions.
+            Exs["org.apache.ignite.services.ServiceDeploymentException"] = (i, m, e) => new ServiceDeploymentException(m, e);
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/ServiceProxySerializer.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/ServiceProxySerializer.cs
index 8e44360ff7b65..bd15a7cabaef0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/ServiceProxySerializer.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/ServiceProxySerializer.cs
@@ -154,6 +154,42 @@ public static object ReadInvocationResult(IBinaryStream stream, Marshaller marsh
                                                  "Examine InnerException for details.", (Exception) err);
         }
 
+        /// <summary>
+        /// Reads service deployment result.
+        /// </summary>
+        /// <param name="stream">Stream.</param>
+        /// <param name="marsh">Marshaller.</param>
+        /// <param name="keepBinary">Binary flag.</param>
+        /// <returns>
+        /// Method invocation result, or exception in case of error.
+        /// </returns>
+        public static void ReadDeploymentResult(IBinaryStream stream, Marshaller marsh, bool keepBinary)
+        {
+            Debug.Assert(stream != null);
+            Debug.Assert(marsh != null);
+
+            var mode = keepBinary ? BinaryMode.ForceBinary : BinaryMode.Deserialize;
+
+            var reader = marsh.StartUnmarshal(stream, mode);
+
+            object err;
+
+            BinaryUtils.ReadInvocationResult(reader, out err);
+
+            if (err == null)
+            {
+                return;
+            }
+
+            var binErr = err as IBinaryObject;
+
+            throw binErr != null
+                ? new ServiceDeploymentException("Service deployment failed with a binary error. " +
+                                                 "Examine BinaryCause for details.", binErr)
+                : new ServiceDeploymentException("Service deployment failed with an exception. " +
+                                                 "Examine InnerException for details.", (Exception) err);
+        }
+
         /// <summary>
         /// Writes the argument in platform-compatible format.
         /// </summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs
index 88d2a76f4dbf9..2f309313f7f9e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs
@@ -25,6 +25,7 @@ namespace Apache.Ignite.Core.Impl.Services
     using Apache.Ignite.Core.Binary;
     using Apache.Ignite.Core.Cluster;
     using Apache.Ignite.Core.Impl.Binary;
+    using Apache.Ignite.Core.Impl.Binary.IO;
     using Apache.Ignite.Core.Impl.Common;
     using Apache.Ignite.Core.Impl.Unmanaged;
     using Apache.Ignite.Core.Services;
@@ -204,13 +205,13 @@ public void DeployMultiple(string name, IService service, int totalCount, int ma
             IgniteArgumentCheck.NotNullOrEmpty(name, "name");
             IgniteArgumentCheck.NotNull(service, "service");
 
-            DoOutOp(OpDeployMultiple, w =>
+            DoOutInOp(OpDeployMultiple, w =>
             {
                 w.WriteString(name);
                 w.WriteObject(service);
                 w.WriteInt(totalCount);
                 w.WriteInt(maxPerNodeCount);
-            });
+            }, s => ReadDeploymentResult(s));
         }
 
         /** <inheritDoc /> */
@@ -225,7 +226,7 @@ public Task DeployMultipleAsync(string name, IService service, int totalCount, i
                 w.WriteObject(service);
                 w.WriteInt(totalCount);
                 w.WriteInt(maxPerNodeCount);
-            });
+            }, _keepBinary, s => ReadDeploymentResult(s));
         }
 
         /** <inheritDoc /> */
@@ -233,7 +234,7 @@ public void Deploy(ServiceConfiguration configuration)
         {
             IgniteArgumentCheck.NotNull(configuration, "configuration");
 
-            DoOutOp(OpDeploy, w => WriteServiceConfiguration(configuration, w));
+            DoOutInOp(OpDeploy, w => WriteServiceConfiguration(configuration, w), s => ReadDeploymentResult(s));
         }
 
         /** <inheritDoc /> */
@@ -241,7 +242,8 @@ public Task DeployAsync(ServiceConfiguration configuration)
         {
             IgniteArgumentCheck.NotNull(configuration, "configuration");
 
-            return DoOutOpAsync(OpDeployAsync, w => WriteServiceConfiguration(configuration, w));
+            return DoOutOpAsync(OpDeployAsync, w => WriteServiceConfiguration(configuration, w), 
+                _keepBinary, ReadDeploymentResult);
         }
 
         /** <inheritDoc /> */
@@ -400,5 +402,22 @@ private static void WriteServiceConfiguration(ServiceConfiguration configuration
             else
                 w.WriteObject<object>(null);
         }
+
+        /// <summary>
+        /// Reads the deployment result.
+        /// </summary>
+        private object ReadDeploymentResult(BinaryReader r)
+        {
+            return r != null ? ReadDeploymentResult(r.Stream) : null;
+        }
+
+        /// <summary>
+        /// Reads the deployment result.
+        /// </summary>
+        private object ReadDeploymentResult(IBinaryStream s)
+        {
+            ServiceProxySerializer.ReadDeploymentResult(s, Marshaller, _keepBinary);
+            return null;
+        }
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs
index cc205e829d476..cd642ad1a27f6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs
@@ -942,22 +942,47 @@ private void EventFilterDestroy(void* target, long ptr)
 
         #region IMPLEMENTATION: SERVICES
 
+        [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes",
+            Justification = "User processor can throw any exception")]
         private long ServiceInit(void* target, long memPtr)
         {
             return SafeCall(() =>
             {
                 using (var stream = IgniteManager.Memory.Get(memPtr).GetStream())
                 {
-                    var reader = _ignite.Marshaller.StartUnmarshal(stream);
+                    try
+                    {
+                        var reader = _ignite.Marshaller.StartUnmarshal(stream);
 
-                    bool srvKeepBinary = reader.ReadBoolean();
-                    var svc = reader.ReadObject<IService>();
+                        var srvKeepBinary = reader.ReadBoolean();
+                        var svc = reader.ReadObject<IService>();
 
-                    ResourceProcessor.Inject(svc, _ignite);
+                        ResourceProcessor.Inject(svc, _ignite);
 
-                    svc.Init(new ServiceContext(_ignite.Marshaller.StartUnmarshal(stream, srvKeepBinary)));
+                        svc.Init(new ServiceContext(_ignite.Marshaller.StartUnmarshal(stream, srvKeepBinary)));
 
-                    return _handleRegistry.Allocate(svc);
+                        stream.Reset();
+
+                        stream.WriteBool(true); // Success.
+
+                        stream.SynchronizeOutput();
+
+                        return _handleRegistry.Allocate(svc);
+                    }
+                    catch (Exception e)
+                    {
+                        stream.Reset();
+
+                        var writer = _ignite.Marshaller.StartMarshal(stream);
+
+                        BinaryUtils.WriteInvocationResult(writer, false, e);
+
+                        _ignite.Marshaller.FinishMarshal(writer);
+
+                        stream.SynchronizeOutput();
+
+                        return 0;
+                    }
                 }
             });
         }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Services/ServiceDeploymentException.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Services/ServiceDeploymentException.cs
new file mode 100644
index 0000000000000..825f91e5b3e5e
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Services/ServiceDeploymentException.cs
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Core.Services
+{
+    using System;
+    using System.Diagnostics.CodeAnalysis;
+    using System.Runtime.Serialization;
+    using Apache.Ignite.Core.Binary;
+    using Apache.Ignite.Core.Common;
+
+    /// <summary>
+    /// Indicates an error during Grid Services deployment.
+    /// </summary>
+    [Serializable]
+    public class ServiceDeploymentException : IgniteException
+    {
+        /** Serializer key. */
+        private const string KeyBinaryCause = "BinaryCause";
+
+        /** Cause. */
+        private readonly IBinaryObject _binaryCause;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ServiceDeploymentException"/> class.
+        /// </summary>
+        public ServiceDeploymentException()
+        {
+            // No-op.
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ServiceDeploymentException"/> class.
+        /// </summary>
+        /// <param name="message">The message that describes the error.</param>
+        public ServiceDeploymentException(string message) : base(message)
+        {
+            // No-op.
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ServiceDeploymentException"/> class.
+        /// </summary>
+        /// <param name="message">The message.</param>
+        /// <param name="cause">The cause.</param>
+        public ServiceDeploymentException(string message, Exception cause) : base(message, cause)
+        {
+            // No-op.
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ServiceDeploymentException"/> class.
+        /// </summary>
+        /// <param name="message">The message.</param>
+        /// <param name="binaryCause">The binary cause.</param>
+        public ServiceDeploymentException(string message, IBinaryObject binaryCause)
+            : base(message)
+        {
+            _binaryCause = binaryCause;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="IgniteException"/> class.
+        /// </summary>
+        /// <param name="info">Serialization information.</param>
+        /// <param name="ctx">Streaming context.</param>
+        protected ServiceDeploymentException(SerializationInfo info, StreamingContext ctx)
+            : base(info, ctx)
+        {
+            _binaryCause = (IBinaryObject)info.GetValue(KeyBinaryCause, typeof(IBinaryObject));
+        }
+
+        /// <summary>
+        /// Gets the binary cause.
+        /// </summary>
+        public IBinaryObject BinaryCause
+        {
+            get { return _binaryCause; }
+        }
+
+        /// <summary>
+        /// When overridden in a derived class, sets the <see cref="SerializationInfo" />
+        /// with information about the exception.
+        /// </summary>
+        /// <param name="info">The <see cref="SerializationInfo" /> that holds the serialized object data
+        /// about the exception being thrown.</param>
+        /// <param name="context">The <see cref="StreamingContext" /> that contains contextual information
+        /// about the source or destination.</param>
+        [SuppressMessage("Microsoft.Design", "CA1062:Validate arguments of public methods")]
+        public override void GetObjectData(SerializationInfo info, StreamingContext context)
+        {
+            info.AddValue(KeyBinaryCause, _binaryCause);
+
+            base.GetObjectData(info, context);
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.sln.DotSettings b/modules/platforms/dotnet/Apache.Ignite.sln.DotSettings
index 078e9fbdb9dbb..9d5b728a9c554 100644
--- a/modules/platforms/dotnet/Apache.Ignite.sln.DotSettings
+++ b/modules/platforms/dotnet/Apache.Ignite.sln.DotSettings
@@ -7,4 +7,5 @@
 	<s:Boolean x:Key="/Default/CodeStyle/CSharpUsing/AddImportsToDeepestScope/@EntryValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeStyle/CSharpUsing/QualifiedUsingAtNestedScope/@EntryValue">True</s:Boolean>
 	<s:String x:Key="/Default/CodeInspection/Highlighting/InspectionSeverities/=ConvertClosureToMethodGroup/@EntryIndexedValue">DO_NOT_SHOW</s:String>
+	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002EXml_002ECodeStyle_002EFormatSettingsUpgrade_002EXmlMoveToCommonFormatterSettingsUpgrade/@EntryIndexedValue">True</s:Boolean>
 </wpf:ResourceDictionary>
\ No newline at end of file

From d3be7960878a302d2b156bba89be440a49b7c4ef Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Thu, 7 Sep 2017 14:47:05 +0300
Subject: [PATCH 362/516] ignite-6289 Remove "allOrNone" flag from
 IgniteServices#deployAll method

---
 .../org/apache/ignite/IgniteServices.java     |  19 +--
 .../ignite/internal/IgniteServicesImpl.java   |   6 +-
 .../GridServiceDeploymentCompoundFuture.java  | 129 ++---------------
 .../service/GridServiceProcessor.java         |  64 +++-----
 .../service/PreparedConfigurations.java       |   8 +-
 ...rviceDeploymentCompoundFutureSelfTest.java | 121 +---------------
 ...idServiceProcessorBatchDeploySelfTest.java | 137 +++++-------------
 7 files changed, 81 insertions(+), 403 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
index 369baebfb6237..edcecd34738ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
@@ -286,24 +286,21 @@ public void deployMultiple(String name, Service svc, int totalCnt, int maxPerNod
 
     /**
      * Deploys multiple services described by provided configurations. Depending on specified parameters, multiple
-     * instances of the same service may be deployed (see {@link ServiceConfiguration}).
-     * Whenever topology changes, Ignite will automatically rebalance
-     * the deployed services within cluster to make sure that each node will end up with
-     * about equal number of deployed instances whenever possible.
+     * instances of the same service may be deployed (see {@link ServiceConfiguration}). Whenever topology changes,
+     * Ignite will automatically rebalance the deployed services within cluster to make sure that each node will end up
+     * with about equal number of deployed instances whenever possible.
      *
-     * If deployment fails, then {@link ServiceDeploymentException} containing a list of failed services will be
-     * thrown. It is guaranteed that all services that were provided to this method and are not present in the list of
-     * failed services are successfully deployed by the moment of the exception being thrown.
+     * If deployment of some of the provided services fails, then {@link ServiceDeploymentException} containing a list
+     * of failed services will be thrown. It is guaranteed that all services that were provided to this method and are
+     * not present in the list of failed services are successfully deployed by the moment of the exception being thrown.
+     * Note that if exception is thrown, then partial deployment may have occurred.
      *
      * @param cfgs {@link Collection} of service configurations to be deployed.
-     * @param allOrNone Specifies behavior in case when errors during deployment occur. If {@code true}, then two
-     * outcomes are possible: either all services will be deployed, or none of them. If {@code false}, then partial
-     * deployments are permitted.
      * @throws ServiceDeploymentException If failed to deploy services.
      * @see IgniteServices#deploy(ServiceConfiguration)
      */
     @IgniteAsyncSupported
-    public void deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) throws ServiceDeploymentException;
+    public void deployAll(Collection<ServiceConfiguration> cfgs) throws ServiceDeploymentException;
 
     /**
      * Cancels service deployment. If a service with specified name was deployed on the grid,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
index a51f3f1ba95fd..58b3a2a96ff1c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
@@ -151,17 +151,17 @@ public IgniteServicesImpl(GridKernalContext ctx, ClusterGroupAdapter prj, boolea
     @Override public void deploy(ServiceConfiguration cfg) {
         A.notNull(cfg, "cfg");
 
-        deployAll(Collections.singleton(cfg), false);
+        deployAll(Collections.singleton(cfg));
     }
 
     /** {@inheritDoc} */
-    @Override public void deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+    @Override public void deployAll(Collection<ServiceConfiguration> cfgs) {
         A.notNull(cfgs, "cfgs");
 
         guard();
 
         try {
-            saveOrGet(ctx.service().deployAll(cfgs, allOrNone));
+            saveOrGet(ctx.service().deployAll(cfgs));
         }
         catch (IgniteCheckedException e) {
             throw U.convertException(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
index bbf03702a226f..45ccc24b9fa86 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFuture.java
@@ -20,140 +20,51 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.services.ServiceDeploymentException;
 import org.jetbrains.annotations.Nullable;
 
 /**
- * Service deployment compound future, {@code allOrNone} parameter specifies failing policy.
- * <p>
- * If {@code allOrNone} parameter is set to {@code false}, then this future waits for completion of all child futures.
- * If any exceptions are thrown during deployment, then {@link IgniteCheckedException} with {@link
- * ServiceDeploymentException} as a cause will be thrown from {@link IgniteInternalFuture#get get()} method after all
- * futures complete or fail. Inner exception will contain configurations of failed services.
+ * Service deployment compound future. If any exceptions are thrown during deployment, then {@link
+ * IgniteCheckedException} with {@link ServiceDeploymentException} as a cause will be thrown from {@link
+ * IgniteInternalFuture#get get()} method after all futures complete or fail. Inner exception will contain
+ * configurations of failed services.
  */
 public class GridServiceDeploymentCompoundFuture extends GridCompoundFuture<Object, Object> {
-    /** */
-    private final boolean allOrNone;
-
-    /** Kernal context. */
-    private final GridKernalContext ctx;
-
-    /** Logger. */
-    private final IgniteLogger log;
-
     /** Names of services written to cache during current deployment. */
     private Collection<String> svcsToRollback;
 
     /** */
     private volatile ServiceDeploymentException err;
 
-    /**
-     * @param allOrNone Failing policy.
-     * @param ctx Kernal context.
-     */
-    GridServiceDeploymentCompoundFuture(boolean allOrNone, GridKernalContext ctx) {
-        this.allOrNone = allOrNone;
-        this.ctx = ctx;
-        this.log = ctx.log(getClass());
-    }
-
     /** {@inheritDoc} */
     @Override protected boolean processFailure(Throwable err, IgniteInternalFuture<Object> fut) {
         assert fut instanceof GridServiceDeploymentFuture : fut;
 
         GridServiceDeploymentFuture depFut = (GridServiceDeploymentFuture)fut;
 
-        if (allOrNone) {
-            if (initialized()) {
-                onDone(new IgniteCheckedException(
-                    new ServiceDeploymentException("Failed to deploy provided services.", err, getConfigurations())));
-            }
-            else {
-                synchronized (this) {
-                    if (this.err == null) {
-                        this.err = new ServiceDeploymentException("Failed to deploy provided services.", err,
-                            new ArrayList<ServiceConfiguration>());
-                    }
-                    else
-                        this.err.addSuppressed(err);
-                }
+        synchronized (this) {
+            if (this.err == null) {
+                this.err = new ServiceDeploymentException("Failed to deploy some services.",
+                    new ArrayList<ServiceConfiguration>());
             }
-        }
-        else {
-            synchronized (this) {
-                if (this.err == null)
-                    this.err = new ServiceDeploymentException("Failed to deploy some services.",
-                        new ArrayList<ServiceConfiguration>());
 
-                this.err.getFailedConfigurations().add(depFut.configuration());
-                this.err.addSuppressed(err);
-            }
+            this.err.getFailedConfigurations().add(depFut.configuration());
+            this.err.addSuppressed(err);
         }
 
         return true;
     }
 
-    /**
-     * Marks this future as initialized. Will complete with error if failures before initialization occurred and
-     * all-or-none policy is followed.
-     */
-    public void serviceDeploymentMarkInitialized() {
-        if (allOrNone && this.err != null) {
-            this.err.getFailedConfigurations().addAll(getConfigurations());
-
-            onDone(new IgniteCheckedException(this.err));
-        }
-        else
-            super.markInitialized();
-    }
-
     /** {@inheritDoc} */
-    @Override protected boolean onDone(@Nullable final Object res, @Nullable Throwable err, final boolean cancel) {
-        final Throwable resErr;
-
+    @Override protected boolean onDone(@Nullable Object res, @Nullable Throwable err, boolean cancel) {
         if (err == null && this.err != null)
-            resErr = new IgniteCheckedException(this.err);
-        else
-            resErr = err;
-
-        if (allOrNone && this.err != null && svcsToRollback != null) {
-            U.warn(log, "Failed to deploy provided services. The following services will be cancelled:" + svcsToRollback);
-
-            IgniteInternalFuture<?> fut = ctx.service().cancelAll(svcsToRollback);
-
-            /*
-            Can not call fut.get() since it is possible we are in system pool now and
-            fut also should be completed from system pool.
-             */
-            fut.listen(new IgniteInClosure<IgniteInternalFuture>() {
-                @Override public void apply(IgniteInternalFuture fut) {
-                    try {
-                        fut.get();
-                    }
-                    catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to cancel deployed services.", e);
-                    }
-                    finally {
-                        svcsToRollback = null;
-                    }
-
-                    GridServiceDeploymentCompoundFuture.super.onDone(res, resErr, cancel);
-                }
-            });
-
-            return false;
-        }
+            err = new IgniteCheckedException(this.err);
 
-        return super.onDone(res, resErr, cancel);
+        return super.onDone(res, err, cancel);
     }
 
     /**
@@ -180,18 +91,4 @@ public Collection<String> servicesToRollback() {
         else
             return Collections.emptyList();
     }
-
-    /**
-     * @return Collection of configurations, stored in child futures.
-     */
-    private Collection<ServiceConfiguration> getConfigurations() {
-        Collection<IgniteInternalFuture<Object>> futs = futures();
-
-        List<ServiceConfiguration> cfgs = new ArrayList<>(futs.size());
-
-        for (IgniteInternalFuture<Object> fut : futs)
-            cfgs.add(((GridServiceDeploymentFuture)fut).configuration());
-
-        return cfgs;
-    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 2a30bad8d4950..8c9ca053baec1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -305,7 +305,7 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
                     c.setNodeFilter(ctx.cluster().get().forServers().predicate());
             }
 
-            deployAll(Arrays.asList(cfgs), true).get();
+            deployAll(Arrays.asList(cfgs)).get();
         }
 
         if (log.isDebugEnabled())
@@ -487,10 +487,9 @@ public IgniteInternalFuture<?> deployKeyAffinitySingleton(String name, Service s
 
     /**
      * @param cfgs Service configurations.
-     * @param allOrNone Failure processing policy.
      * @return Configurations to deploy.
      */
-    private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+    private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceConfiguration> cfgs) {
         List<ServiceConfiguration> cfgsCp = new ArrayList<>(cfgs.size());
 
         ServicesCompatibilityState state = markCompatibilityStateAsUsed();
@@ -543,40 +542,28 @@ private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceCo
             }
 
             if (err != null) {
-                if (allOrNone) {
-                    return new PreparedConfigurations(null,
-                        null,
-                        new IgniteCheckedException(
-                            new ServiceDeploymentException("None of the provided services were deplyed.", err, cfgs)));
-                }
-                else {
-                    if (failedFuts == null)
-                        failedFuts = new ArrayList<>();
+                if (failedFuts == null)
+                    failedFuts = new ArrayList<>();
 
-                    GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+                GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
 
-                    fut.onDone(err);
+                fut.onDone(err);
 
-                    failedFuts.add(fut);
-                }
+                failedFuts.add(fut);
             }
         }
 
-        return new PreparedConfigurations(cfgsCp, failedFuts, null);
+        return new PreparedConfigurations(cfgsCp, failedFuts);
     }
 
     /**
      * @param cfgs Service configurations.
-     * @param allOrNone Failure processing policy.
      * @return Future for deployment.
      */
-    public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs, boolean allOrNone) {
+    public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs) {
         assert cfgs != null;
 
-        PreparedConfigurations srvCfg = prepareServiceConfigurations(cfgs, allOrNone);
-
-        if (srvCfg.err != null)
-            return new GridFinishedFuture<>(srvCfg.err);
+        PreparedConfigurations srvCfg = prepareServiceConfigurations(cfgs);
 
         List<ServiceConfiguration> cfgsCp = srvCfg.cfgs;
 
@@ -591,7 +578,7 @@ public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs,
         GridServiceDeploymentCompoundFuture res;
 
         while (true) {
-            res = new GridServiceDeploymentCompoundFuture(allOrNone, ctx);
+            res = new GridServiceDeploymentCompoundFuture();
 
             if (ctx.deploy().enabled())
                 ctx.cache().context().deploy().ignoreOwnership(true);
@@ -608,16 +595,8 @@ else if (cfgsCp.size() > 1) {
                             catch (IgniteCheckedException e) {
                                 if (X.hasCause(e, ClusterTopologyCheckedException.class))
                                     throw e; // Retry.
-
-                                if (allOrNone) {
-                                    for (String name : res.servicesToRollback())
-                                        depFuts.remove(name).onDone(e);
-
-                                    res.onDone(new IgniteCheckedException(new ServiceDeploymentException(
-                                        "Failed to deploy provided services.", e, cfgs)));
-
-                                    return res;
-                                }
+                                else
+                                    U.error(log, e.getMessage());
                             }
                         }
 
@@ -651,7 +630,7 @@ else if (cfgsCp.size() > 1) {
         if (ctx.clientDisconnected()) {
             IgniteClientDisconnectedCheckedException err =
                 new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
-                "Failed to deploy services, client node disconnected: " + cfgs);
+                    "Failed to deploy services, client node disconnected: " + cfgs);
 
             for (String name : res.servicesToRollback()) {
                 GridServiceDeploymentFuture fut = depFuts.remove(name);
@@ -668,7 +647,7 @@ else if (cfgsCp.size() > 1) {
                 res.add(fut, false);
         }
 
-        res.serviceDeploymentMarkInitialized();
+        res.markInitialized();
 
         return res;
     }
@@ -705,12 +684,8 @@ private void writeServiceToCache(GridServiceDeploymentCompoundFuture res, Servic
 
             if (dep != null) {
                 if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
-                    String err = "Failed to deploy service (service already exists with different " +
-                        "configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']';
-
-                    U.error(log, err);
-
-                    throw new IgniteCheckedException(err);
+                    throw new IgniteCheckedException("Failed to deploy service (service already exists with " +
+                        "different configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']');
                 }
                 else {
                     res.add(fut, false);
@@ -755,7 +730,7 @@ private void writeServiceToCache(GridServiceDeploymentCompoundFuture res, Servic
     public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
         A.notNull(cfg, "cfg");
 
-        return deployAll(Collections.singleton(cfg), false);
+        return deployAll(Collections.singleton(cfg));
     }
 
     /**
@@ -788,7 +763,8 @@ public IgniteInternalFuture<?> cancel(String name) {
                 if (X.hasCause(e, ClusterTopologyCheckedException.class)) {
                     if (log.isDebugEnabled())
                         log.debug("Topology changed while cancelling service (will retry): " + e.getMessage());
-                } else {
+                }
+                else {
                     U.error(log, "Failed to undeploy service: " + name, e);
 
                     return new GridFinishedFuture<>(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
index a581e159348a8..dc41c22311722 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/PreparedConfigurations.java
@@ -31,19 +31,13 @@ class PreparedConfigurations {
     /** */
     final List<GridServiceDeploymentFuture> failedFuts;
 
-    /** */
-    final Exception err;
-
     /**
      * @param cfgs Configurations to deploy.
      * @param failedFuts Finished futures for failed configurations.
-     * @param err Error if need to stop deploy.
      */
-    PreparedConfigurations(List<ServiceConfiguration> cfgs, List<GridServiceDeploymentFuture> failedFuts,
-        Exception err) {
+    PreparedConfigurations(List<ServiceConfiguration> cfgs, List<GridServiceDeploymentFuture> failedFuts) {
         this.cfgs = cfgs;
         this.failedFuts = failedFuts;
-        this.err = err;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
index ca95198cf2fc9..bcc050400835b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceDeploymentCompoundFutureSelfTest.java
@@ -22,9 +22,7 @@
 import java.util.List;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
-import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.services.ServiceConfiguration;
@@ -33,16 +31,6 @@
 
 /** */
 public class GridServiceDeploymentCompoundFutureSelfTest extends GridCommonAbstractTest {
-    /** */
-    private static GridKernalContext ctx;
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        IgniteKernal kernal = (IgniteKernal)startGrid(0);
-
-        ctx = kernal.context();
-    }
-
     /** {@inheritDoc} */
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();
@@ -51,8 +39,8 @@ public class GridServiceDeploymentCompoundFutureSelfTest extends GridCommonAbstr
     /**
      * @throws Exception If failed.
      */
-    public void testWaitForCompletionOnFailingFuturePartial() throws Exception {
-        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(false, ctx);
+    public void testWaitForCompletionOnFailingFuture() throws Exception {
+        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture();
 
         int failingFutsNum = 2;
 
@@ -80,7 +68,7 @@ public void testWaitForCompletionOnFailingFuturePartial() throws Exception {
             compFut.add(fut);
         }
 
-        compFut.serviceDeploymentMarkInitialized();
+        compFut.markInitialized();
 
         List<Exception> causes = new ArrayList<>();
 
@@ -125,109 +113,6 @@ public void testWaitForCompletionOnFailingFuturePartial() throws Exception {
         }
     }
 
-    /**
-     * @throws Exception if failed.
-     */
-    public void testFailAllAfterInitialized() throws Exception {
-        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(true, ctx);
-
-        ServiceConfiguration failingCfg = config("Failed");
-
-        GridServiceDeploymentFuture failingFut = new GridServiceDeploymentFuture(failingCfg);
-
-        compFut.add(failingFut);
-
-        int futsNum = 5;
-
-        List<ServiceConfiguration> cfgs = new ArrayList<>(futsNum + 1);
-
-        cfgs.add(failingCfg);
-
-        for (int i = 0; i < futsNum; i++) {
-            ServiceConfiguration cfg = config(String.valueOf(i));
-
-            cfgs.add(cfg);
-
-            compFut.add(new GridServiceDeploymentFuture(cfg));
-        }
-
-        compFut.serviceDeploymentMarkInitialized();
-
-        Exception expCause = new Exception("Test error");
-
-        failingFut.onDone(expCause);
-
-        assertFailAll(compFut, cfgs, expCause);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testFailAllBeforeInitialized() throws Exception {
-        GridServiceDeploymentCompoundFuture compFut = new GridServiceDeploymentCompoundFuture(true, ctx);
-
-        ServiceConfiguration failingCfg = config("Failed");
-
-        GridServiceDeploymentFuture failingFut = new GridServiceDeploymentFuture(failingCfg);
-
-        Exception expCause = new Exception("Test error");
-
-        failingFut.onDone(expCause);
-
-        compFut.add(failingFut);
-
-        assertFalse(compFut.isDone());
-
-        int futsNum = 5;
-
-        List<ServiceConfiguration> cfgs = new ArrayList<>(futsNum + 1);
-
-        cfgs.add(failingCfg);
-
-        for (int i = 0; i < futsNum; i++) {
-            ServiceConfiguration cfg = config(String.valueOf(i));
-
-            cfgs.add(cfg);
-
-            compFut.add(new GridServiceDeploymentFuture(cfg));
-        }
-
-        compFut.serviceDeploymentMarkInitialized();
-
-        assertFailAll(compFut, cfgs, expCause);
-    }
-
-    /**
-     * Try waiting for the future completion and check that a proper exception is thrown.
-     *
-     * @param fut Future.
-     * @param expCfgs Expected cfgs.
-     * @param expCause Expected cause.
-     */
-    private void assertFailAll(GridServiceDeploymentCompoundFuture fut, Collection<ServiceConfiguration> expCfgs,
-        Exception expCause) {
-        try {
-            fut.get();
-
-            fail("Should never reach here.");
-        }
-        catch (IgniteCheckedException ce) {
-            log.info("Expected exception: " + ce.getMessage());
-
-            IgniteException e = U.convertException(ce);
-
-            assertTrue(e instanceof ServiceDeploymentException);
-
-            assertEqualsCollections(expCfgs, ((ServiceDeploymentException)e).getFailedConfigurations());
-
-            Throwable actCause = e.getCause();
-
-            assertTrue(actCause instanceof IgniteCheckedException);
-
-            assertEquals(expCause, actCause.getCause());
-        }
-    }
-
     /**
      * @param name Name.
      * @return Dummy configuration with a specified name.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
index 337e69565968b..ab012b37ea551 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorBatchDeploySelfTest.java
@@ -100,7 +100,7 @@ public void testDeployAll() throws Exception {
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(cfgs, false);
+        client.services().deployAll(cfgs);
 
         assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
 
@@ -121,7 +121,7 @@ public void testDeployAllAsync() throws Exception {
 
         IgniteServices services =  client.services().withAsync();
 
-        services.deployAll(cfgs, false);
+        services.deployAll(cfgs);
 
         services.future().get();
 
@@ -131,7 +131,7 @@ public void testDeployAllAsync() throws Exception {
     }
 
     /**
-     * TODO: enable when IGNITE-6259 is fixed
+     * TODO: enable when IGNITE-6259 is fixed.
      *
      * @throws Exception If failed.
      */
@@ -170,7 +170,7 @@ public void _testDeployAllTopologyChange() throws Exception {
             while (from < numServices) {
                 int to = Math.min(numServices, from + batchSize);
 
-                services.deployAll(cfgs.subList(from, to), false);
+                services.deployAll(cfgs.subList(from, to));
 
                 services.future().get(5000);
 
@@ -189,7 +189,7 @@ public void _testDeployAllTopologyChange() throws Exception {
     }
 
     /**
-     * TODO: enable when IGNITE-6259 is fixed
+     * TODO: enable when IGNITE-6259 is fixed.
      *
      * @throws Exception If failed.
      */
@@ -239,7 +239,7 @@ public void _testDeployAllTopologyChangeFail() throws Exception {
                 failingCfgs.add(failingCfg);
 
                 try {
-                    services.deployAll(cfgsBatch, false);
+                    services.deployAll(cfgsBatch);
 
                     services.future().get(5000);
 
@@ -274,37 +274,22 @@ public void _testDeployAllTopologyChangeFail() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testDeployAllFailAll() throws Exception {
-        deployAllFail(false, true);
+    public void testDeployAllFail() throws Exception {
+        deployAllFail(false);
     }
 
     /**
      * @throws Exception If failed.
      */
-    public void testDeployAllPartial() throws Exception {
-        deployAllFail(false, false);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testDeployAllAsyncFailAll() throws Exception {
-        deployAllFail(true, true);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testDeployAllAsyncFailPartial() throws Exception {
-        deployAllFail(true, false);
+    public void testDeployAllAsyncFail() throws Exception {
+        deployAllFail(true);
     }
 
     /**
      * @param async If {@code true}, then asynchronous method of deployment will be performed.
-     * @param allOrNone Failing strategy.
      * @throws Exception If failed.
      */
-    private void deployAllFail(boolean async, boolean allOrNone) throws Exception {
+    private void deployAllFail(boolean async) throws Exception {
         Ignite client = grid(CLIENT_NODE_NAME);
 
         CountDownLatch latch = new CountDownLatch(NUM_SERVICES - 1);
@@ -317,20 +302,11 @@ private void deployAllFail(boolean async, boolean allOrNone) throws Exception {
 
         failingCfg.setName(null);
 
-        assertFailingDeploy(client, async, allOrNone, cfgs, failingCfg);
+        assertFailingDeploy(client, async, cfgs, failingCfg);
 
-        if (allOrNone) {
-            assertFalse("Some of the services were deployed.", latch.await(2, TimeUnit.SECONDS));
-
-            assertEquals(NUM_SERVICES - 1, latch.getCount());
-
-            assertTrue(client.services().serviceDescriptors().isEmpty());
-        }
-        else {
-            assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
+        assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
 
-            assertDeployedServices(client, cfgs.subList(0, cfgs.size() - 1));
-        }
+        assertDeployedServices(client, cfgs.subList(0, cfgs.size() - 1));
     }
 
     /**
@@ -351,8 +327,8 @@ public void testClashingNames() throws Exception {
         IgniteServices svcs1 = client.services().withAsync();
         IgniteServices svcs2 = client.services().withAsync();
 
-        svcs1.deployAll(fstBatch, false);
-        svcs2.deployAll(sndBatch, false);
+        svcs1.deployAll(fstBatch);
+        svcs2.deployAll(sndBatch);
 
         svcs1.future().get();
         svcs2.future().get();
@@ -365,32 +341,12 @@ public void testClashingNames() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testClashingNamesFailAll() throws Exception {
-        clashingNamesFail(true);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testClashingNamesPartial() throws Exception {
-        clashingNamesFail(false);
-    }
-
-    /**
-     * @param allOrNone Failing strategy.
-     * @throws Exception If failed.
-     */
-    private void clashingNamesFail(boolean allOrNone) throws Exception {
+    public void testClashingNamesFail() throws Exception {
         Ignite client = grid(CLIENT_NODE_NAME);
 
         List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
 
-        int numDepSvcs;
-
-        if (allOrNone)
-            numDepSvcs = NUM_SERVICES / 2;
-        else
-            numDepSvcs = NUM_SERVICES - 1;
+        int numDepSvcs = NUM_SERVICES - 1;
 
         CountDownLatch latch = new CountDownLatch(numDepSvcs);
 
@@ -401,13 +357,13 @@ private void clashingNamesFail(boolean allOrNone) throws Exception {
 
         IgniteServices services = client.services().withAsync();
 
-        services.deployAll(fstBatch, false);
+        services.deployAll(fstBatch);
 
         ServiceConfiguration failingCfg = cfgs.get(NUM_SERVICES - 1);
 
         failingCfg.setName(null);
 
-        assertFailingDeploy(client, false, allOrNone, sndBatch, failingCfg);
+        assertFailingDeploy(client, false, sndBatch, failingCfg);
 
         services.future().get();
 
@@ -419,33 +375,12 @@ private void clashingNamesFail(boolean allOrNone) throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testClashingNameDifferentConfigFailAll() throws Exception {
-        testClashingNameDifferentConfig(true);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testClashingNameDifferentConfigPartial() throws Exception {
-        testClashingNameDifferentConfig(false);
-    }
-
-    /**
-     * @param allOrNone Failing strategy.
-     * @throws Exception If failed.
-     */
-    private void testClashingNameDifferentConfig(boolean allOrNone) throws Exception {
+    public void testClashingNameDifferentConfig() throws Exception {
         Ignite client = grid(CLIENT_NODE_NAME);
 
         List<ServiceConfiguration> cfgs = getConfigs(client.cluster().forServers().predicate(), NUM_SERVICES);
 
-        int numDepSvcs;
-
-        if (allOrNone)
-            numDepSvcs = NUM_SERVICES / 2;
-        else
-            numDepSvcs = NUM_SERVICES - 1;
-
+        int numDepSvcs = NUM_SERVICES - 1;
 
         CountDownLatch latch = new CountDownLatch(numDepSvcs);
 
@@ -454,7 +389,7 @@ private void testClashingNameDifferentConfig(boolean allOrNone) throws Exception
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(fstBatch, false);
+        client.services().deployAll(fstBatch);
 
         ServiceConfiguration failingCfg = copyService(cfgs.get(NUM_SERVICES - 1));
 
@@ -464,7 +399,7 @@ private void testClashingNameDifferentConfig(boolean allOrNone) throws Exception
 
         sndBatch.add(failingCfg);
 
-        assertFailingDeploy(client, false, allOrNone, sndBatch, failingCfg);
+        assertFailingDeploy(client, false, sndBatch, failingCfg);
 
         assertTrue("Waiting for services deployment timed out.", latch.await(30, TimeUnit.SECONDS));
 
@@ -483,7 +418,7 @@ public void testCancelAll() throws Exception {
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(cfgs, true);
+        client.services().deployAll(cfgs);
 
         latch.await(30, TimeUnit.SECONDS);
 
@@ -504,7 +439,7 @@ public void testCancelAllAsync() throws Exception {
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(cfgs, true);
+        client.services().deployAll(cfgs);
 
         latch.await(30, TimeUnit.SECONDS);
 
@@ -518,7 +453,7 @@ public void testCancelAllAsync() throws Exception {
     }
 
     /**
-     * TODO: enable when IGNITE-6259 is fixed
+     * TODO: enable when IGNITE-6259 is fixed.
      *
      * @throws Exception If failed.
      */
@@ -533,7 +468,7 @@ public void _testCancelAllTopologyChange() throws Exception {
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(cfgs, true);
+        client.services().deployAll(cfgs);
 
         latch.await(30, TimeUnit.SECONDS);
 
@@ -585,7 +520,7 @@ public void testCancelAllClashingNames() throws Exception {
 
         subscribeExeLatch(cfgs, latch);
 
-        client.services().deployAll(cfgs, true);
+        client.services().deployAll(cfgs);
 
         latch.await(30, TimeUnit.SECONDS);
 
@@ -616,11 +551,10 @@ public void testCancelAllClashingNames() throws Exception {
      * @param client Client.
      * @param async If {@code true}, then async version of deploy method will be used.
      * @param cfgs Service configurations.
-     * @param allOrNone Failing policy.
      * @param failingCfg Configuration of the failing service.
      * @throws Exception If failed.
      */
-    private void assertFailingDeploy(Ignite client, boolean async, boolean allOrNone, List<ServiceConfiguration> cfgs,
+    private void assertFailingDeploy(Ignite client, boolean async, List<ServiceConfiguration> cfgs,
         ServiceConfiguration failingCfg) throws Exception {
 
         IgniteServices services = client.services();
@@ -628,26 +562,21 @@ private void assertFailingDeploy(Ignite client, boolean async, boolean allOrNone
         if (async) {
             services = services.withAsync();
 
-            services.deployAll(cfgs, allOrNone);
+            services.deployAll(cfgs);
         }
 
         try {
             if (async)
                 services.future().get();
             else
-                services.deployAll(cfgs, allOrNone);
+                services.deployAll(cfgs);
 
             fail("Should never reach here.");
         }
         catch (ServiceDeploymentException e) {
             info("Expected exception: " + e.getMessage());
 
-            Collection<ServiceConfiguration> expFails;
-
-            if (allOrNone)
-                expFails = cfgs;
-            else
-                expFails = Collections.singleton(failingCfg);
+            Collection<ServiceConfiguration> expFails = Collections.singleton(failingCfg);
 
             Collection<ServiceConfiguration> actFails = e.getFailedConfigurations();
 

From 565ee846490834b74a9cc0f8cd0a28a5494392cd Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 8 Sep 2017 11:20:28 +0300
Subject: [PATCH 363/516] GG-12690 - Deregister local continuous query handlers
 on cache stop.

---
 .../CacheContinuousQueryHandler.java          |  7 ++++++
 .../CacheContinuousQueryManager.java          |  2 +-
 .../continuous/GridContinuousProcessor.java   | 24 +++++++++++++++++++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 0cdbec5013b15..ad11013678db5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -227,6 +227,13 @@ public void internal(boolean internal) {
         this.internal = internal;
     }
 
+    /**
+     * @return Internal query.
+     */
+    public boolean internal() {
+        return internal;
+    }
+
     /**
      * @param notifyExisting Notify existing.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 12b02f0fa9386..b3f7d6e93f7b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -839,7 +839,7 @@ GridContinuousHandler.RegisterStatus registerListener(UUID lsnrId,
      * @param internal Internal flag.
      * @param id Listener ID.
      */
-    void unregisterListener(boolean internal, UUID id) {
+    public void unregisterListener(boolean internal, UUID id) {
         CacheContinuousQueryListener lsnr;
 
         if (internal) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 50d29d1f8de6e..6b1d0beaf02e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -61,6 +61,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -568,6 +569,29 @@ public void onCacheStop(GridCacheContext ctx) {
             if (hnd.isQuery() && F.eq(ctx.name(), hnd.cacheName()))
                 it.remove();
         }
+
+        // Deregister local handlers.
+        Iterator<Map.Entry<UUID, LocalRoutineInfo>> it2 = locInfos.entrySet().iterator();
+
+        while (it2.hasNext()) {
+            Map.Entry<UUID, LocalRoutineInfo> entry = it2.next();
+
+            GridContinuousHandler hnd = entry.getValue().hnd;
+
+            if (hnd.isQuery() && F.eq(ctx.name(), hnd.cacheName())) {
+                it2.remove();
+
+                assert hnd instanceof CacheContinuousQueryHandler : hnd;
+
+                CacheContinuousQueryHandler hnd0 = (CacheContinuousQueryHandler)hnd;
+
+                unregisterHandler(entry.getKey(), hnd, true);
+
+                CacheContinuousQueryManager qryMgr = ctx.continuousQueries();
+
+                qryMgr.unregisterListener(hnd0.internal(), entry.getKey());
+            }
+        }
     }
 
     /**

From 1c07cc9babf2e77736cd1caa44ddf944eeea3cf1 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 8 Sep 2017 14:37:03 +0300
Subject: [PATCH 364/516] ignite-12754 fix for
 IgniteCacheCreatePutTest.testUpdatesAndCacheStart.

fix is based on the following commit: 05dd08b993e2d7f88176c051463b178431714f85
---
 .../processors/cache/distributed/dht/GridDhtCacheAdapter.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 2fa934b79db71..417eb3536536f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -219,7 +219,8 @@ protected GridDhtCacheAdapter(GridCacheContext<K, V> ctx, GridCacheConcurrentMap
     @Override public void onKernalStart() throws IgniteCheckedException {
         super.onKernalStart();
 
-        preldr.onKernalStart();
+        if (preldr != null)
+            preldr.onKernalStart();
     }
 
     /** {@inheritDoc} */

From 9e3e1149173787f7e5501f937907f3fc6ffd6ffe Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 5 Sep 2016 13:43:54 +0300
Subject: [PATCH 365/516] Minor fix for IGNITE-6256: Add localnode to topology
 snapshot.

(cherry picked from commit bb40e57)
---
 .../ignite/spi/discovery/tcp/ServerImpl.java   | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index cfa36f1599590..b1fd7e23f308f 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -458,34 +458,36 @@ else if (log.isInfoEnabled()) {
         U.interrupt(statsPrinter);
         U.join(statsPrinter, log);
 
-        Collection<TcpDiscoveryNode> rmts = null;
+        Collection<TcpDiscoveryNode> nodes = null;
 
         if (!disconnect)
             spi.printStopInfo();
         else {
             spi.getSpiContext().deregisterPorts();
 
-            rmts = ring.visibleRemoteNodes();
+            nodes = ring.visibleNodes();
         }
 
         long topVer = ring.topologyVersion();
 
         ring.clear();
 
-        if (rmts != null && !rmts.isEmpty()) {
-            // This is restart/disconnection and remote nodes are not empty.
-            // We need to fire FAIL event for each.
+        if (nodes != null) {
+            // This is restart/disconnection and we need to fire FAIL event for each remote node.
             DiscoverySpiListener lsnr = spi.lsnr;
 
             if (lsnr != null) {
-                Collection<ClusterNode> processed = new HashSet<>();
+                Collection<ClusterNode> processed = new HashSet<>(nodes.size());
+
+                for (TcpDiscoveryNode n : nodes) {
+                    if(n.isLocal())
+                        continue;
 
-                for (TcpDiscoveryNode n : rmts) {
                     assert n.visible();
 
                     processed.add(n);
 
-                    List<ClusterNode> top = U.arrayList(rmts, F.notIn(processed));
+                    List<ClusterNode> top = U.arrayList(nodes, F.notIn(processed));
 
                     topVer++;
 

From aa264b1474dd964dc46287bd80d9e6e1ba3fb758 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Mon, 11 Sep 2017 15:58:15 +0300
Subject: [PATCH 366/516] GG-12708 fix
 IgniteCacheTxStoreSessionWriteBehindCoalescingTest.testSession

---
 .../IgniteCacheStoreSessionWriteBehindAbstractTest.java         | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java
index 4b0f7c0bb22c0..f8665373c4cb4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java
@@ -104,8 +104,6 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throw
 
         ccfg1.setName(CACHE_NAME1);
 
-        ccfg1.setCacheStoreFactory(singletonFactory(new TestStore()));
-
         cfg.setCacheConfiguration(ccfg0, ccfg1);
 
         return cfg;

From fbd1f2d201a4c6f95ddc485c3d7ff4f8fd5adf8f Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Tue, 13 Jun 2017 19:41:55 +0300
Subject: [PATCH 367/516] Backport test fix: ignite-2.1.1 Extract Ignite
 updates checker to separate class. Fixed GridUpdateNotifier test.

(cherry picked from commit 2b030c0)
---
 .../cluster/GridUpdateNotifier.java           | 50 +++++++-------
 .../cluster/HttpIgniteUpdatesChecker.java     | 65 +++++++++++++++++++
 .../cluster/GridUpdateNotifierSelfTest.java   | 10 ++-
 3 files changed, 97 insertions(+), 28 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
index 5b2edcd94bab3..df956b6e62972 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
@@ -64,8 +64,8 @@ class GridUpdateNotifier {
     /** Sleep milliseconds time for worker thread. */
     private static final int WORKER_THREAD_SLEEP_TIME = 5000;
 
-    /** Url for request version. */
-    private final static String UPDATE_NOTIFIER_URL = "https://ignite.run/update_status_ignite-plain-text.php";
+    /** Default url for request Ignite updates. */
+    private final static String DEFAULT_IGNITE_UPDATES_URL = "https://ignite.run/update_status_ignite-plain-text.php";
 
     /** Grid version. */
     private final String ver;
@@ -103,6 +103,9 @@ class GridUpdateNotifier {
     /** Worker thread to process http request. */
     private final Thread workerThread;
 
+    /** Http client for getting Ignite updates */
+    private final HttpIgniteUpdatesChecker updatesChecker;
+
     /**
      * Creates new notifier with default values.
      *
@@ -111,14 +114,16 @@ class GridUpdateNotifier {
      * @param gw Kernal gateway.
      * @param pluginProviders Kernal gateway.
      * @param reportOnlyNew Whether or not to report only new version.
+     * @param updatesChecker Service for getting Ignite updates
      * @throws IgniteCheckedException If failed.
      */
     GridUpdateNotifier(String gridName, String ver, GridKernalGateway gw, Collection<PluginProvider> pluginProviders,
-        boolean reportOnlyNew) throws IgniteCheckedException {
+                       boolean reportOnlyNew, HttpIgniteUpdatesChecker updatesChecker) throws IgniteCheckedException {
         try {
             this.ver = ver;
             this.gridName = gridName == null ? "null" : gridName;
             this.gw = gw;
+            this.updatesChecker = updatesChecker;
 
             SB pluginsBuilder = new SB();
 
@@ -159,6 +164,14 @@ class GridUpdateNotifier {
         }
     }
 
+    /**
+     * Creates new notifier with default Ignite updates URL
+     */
+    GridUpdateNotifier(String igniteInstanceName, String ver, GridKernalGateway gw, Collection<PluginProvider> pluginProviders,
+                       boolean reportOnlyNew) throws IgniteCheckedException {
+        this(igniteInstanceName, ver, gw, pluginProviders, reportOnlyNew, new HttpIgniteUpdatesChecker(DEFAULT_IGNITE_UPDATES_URL, CHARSET));
+    }
+
     /**
      * Gets system properties.
      *
@@ -313,34 +326,17 @@ private class UpdateChecker extends GridWorker {
                     (!F.isEmpty(vmProps) ? "&vmProps=" + encode(vmProps, CHARSET) : "") +
                         pluginsVers;
 
-                URLConnection conn = new URL(UPDATE_NOTIFIER_URL).openConnection();
-
                 if (!isCancelled()) {
-                    conn.setDoOutput(true);
-                    conn.setRequestProperty("Accept-Charset", CHARSET);
-                    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=" + CHARSET);
-
-                    conn.setConnectTimeout(3000);
-                    conn.setReadTimeout(3000);
-
                     try {
-                        try (OutputStream os = conn.getOutputStream()) {
-                            os.write(postParams.getBytes(CHARSET));
-                        }
-
-                        try (InputStream in = conn.getInputStream()) {
-                            if (in == null)
-                                return;
-
-                            BufferedReader reader = new BufferedReader(new InputStreamReader(in, CHARSET));
+                        String updatesResponse = updatesChecker.getUpdates(postParams);
 
-                            for (String line; (line = reader.readLine()) != null; ) {
-                                if (line.contains("version"))
-                                    latestVer = obtainVersionFrom(line);
-                                else if (line.contains("downloadUrl"))
-                                    downloadUrl = obtainDownloadUrlFrom(line);
-                            }
+                        String[] lines = updatesResponse.split("\n");
 
+                        for (String line : lines) {
+                            if (line.contains("version"))
+                                latestVer = obtainVersionFrom(line);
+                            else if (line.contains("downloadUrl"))
+                                downloadUrl = obtainDownloadUrlFrom(line);
                         }
                     }
                     catch (IOException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
new file mode 100644
index 0000000000000..c052c0997fe2a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
@@ -0,0 +1,65 @@
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.URL;
+import java.net.URLConnection;
+
+/**
+ * This class is responsible for getting Ignite updates information via HTTP
+ */
+public class HttpIgniteUpdatesChecker {
+    /** Url for request updates. */
+    private final String url;
+
+    /** Charset for encoding requests/responses */
+    private final String charset;
+
+    /**
+     * Creates new HTTP Ignite updates checker with following parameters
+     * @param url URL for getting Ignite updates information
+     * @param charset Charset for encoding
+     */
+    HttpIgniteUpdatesChecker(String url, String charset) {
+        this.url = url;
+        this.charset = charset;
+    }
+
+    /**
+     * Gets information about Ignite updates via HTTP
+     * @param updateRequest HTTP Request parameters
+     * @return Information about Ignite updates separated by line endings
+     * @throws IOException If HTTP request was failed
+     */
+    public String getUpdates(String updateRequest) throws IOException {
+        URLConnection conn = new URL(url).openConnection();
+        conn.setDoOutput(true);
+        conn.setRequestProperty("Accept-Charset", charset);
+        conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=" + charset);
+
+        conn.setConnectTimeout(3000);
+        conn.setReadTimeout(3000);
+
+        try (OutputStream os = conn.getOutputStream()) {
+            os.write(updateRequest.getBytes(charset));
+        }
+
+        try (InputStream in = conn.getInputStream()) {
+            if (in == null)
+                return null;
+
+            BufferedReader reader = new BufferedReader(new InputStreamReader(in, charset));
+
+            StringBuilder response = new StringBuilder();
+
+            for (String line; (line = reader.readLine()) != null; ) {
+                response.append(line).append('\n');
+            }
+
+            return response.toString();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java
index 21b91b6b3c53e..1a20f261550ba 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifierSelfTest.java
@@ -29,6 +29,8 @@
 import org.apache.ignite.plugin.PluginProvider;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
 
 /**
  * Update notifier test.
@@ -73,8 +75,14 @@ public class GridUpdateNotifierSelfTest extends GridCommonAbstractTest {
     public void testNotifier() throws Exception {
         String nodeVer = IgniteProperties.get("ignite.version");
 
+        HttpIgniteUpdatesChecker updatesCheckerMock = Mockito.mock(HttpIgniteUpdatesChecker.class);
+
+        // Return current node version and some other info
+        Mockito.when(updatesCheckerMock.getUpdates(Matchers.anyString()))
+                .thenReturn("meta=meta" + "\n" + "version=" + nodeVer + "\n" + "downloadUrl=url");
+
         GridUpdateNotifier ntf = new GridUpdateNotifier(null, nodeVer,
-            TEST_GATEWAY, Collections.<PluginProvider>emptyList(), false);
+            TEST_GATEWAY, Collections.<PluginProvider>emptyList(), false, updatesCheckerMock);
 
         ntf.checkForNewVersion(log);
 

From 168655820615765817b05d8e8dbe9cbae5fd41dd Mon Sep 17 00:00:00 2001
From: Konstantin Dudkov <kdudkov@ya.ru>
Date: Fri, 7 Apr 2017 17:17:04 +0300
Subject: [PATCH 368/516] IGNITE-4876 - Tests should wait for topology change
 on all nodes

(cherry picked from commit 4e80ddf)
---
 .../junits/GridAbstractTest.java              | 53 ++++++++++++++++++-
 1 file changed, 51 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 5c43ecbfc9bb2..a9815483ee453 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -57,16 +57,20 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgnitionEx;
 import org.apache.ignite.internal.binary.BinaryEnumCache;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
 import org.apache.ignite.internal.util.GridClassLoaderCache;
 import org.apache.ignite.internal.util.GridTestClockTimer;
 import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -135,6 +139,9 @@ public abstract class GridAbstractTest extends TestCase {
     /** */
     private static final long DFLT_TEST_TIMEOUT = 5 * 60 * 1000;
 
+    /** */
+    private static final int DFLT_TOP_WAIT_TIMEOUT = 2000;
+
     /** */
     private static final transient Map<Class<?>, TestCounters> tests = new ConcurrentHashMap<>();
 
@@ -862,6 +869,16 @@ protected void stopGrid(@Nullable String gridName) {
      */
     @SuppressWarnings({"deprecation"})
     protected void stopGrid(@Nullable String gridName, boolean cancel) {
+        stopGrid(gridName, cancel, true);
+    }
+
+    /**
+     * @param gridName Ignite instance name.
+     * @param cancel Cancel flag.
+     * @param awaitTop Await topology change flag.
+     */
+    @SuppressWarnings({"deprecation"})
+    protected void stopGrid(@Nullable String gridName, boolean cancel, boolean awaitTop) {
         try {
             Ignite ignite = grid(gridName);
 
@@ -874,6 +891,9 @@ protected void stopGrid(@Nullable String gridName, boolean cancel) {
                 G.stop(gridName, cancel);
             else
                 IgniteProcessProxy.stop(gridName, cancel);
+
+            if (awaitTop)
+                awaitTopologyChange();
         }
         catch (IllegalStateException ignored) {
             // Ignore error if grid already stopped.
@@ -908,10 +928,10 @@ protected void stopAllGrids(boolean cancel) {
             }
 
             for (Ignite g : clients)
-                stopGrid(g.name(), cancel);
+                stopGrid(g.name(), cancel, false);
 
             for (Ignite g : srvs)
-                stopGrid(g.name(), cancel);
+                stopGrid(g.name(), cancel, false);
 
             assert G.allGrids().isEmpty();
         }
@@ -1894,6 +1914,35 @@ public static IgniteEx grid(String name, boolean remote, boolean thisRemote) {
         }
     }
 
+    /**
+     *
+     * @throws IgniteInterruptedCheckedException
+     */
+    public void awaitTopologyChange() throws IgniteInterruptedCheckedException {
+        for (Ignite g : G.allGrids()) {
+            final GridKernalContext ctx = ((IgniteKernal)g).context();
+
+            if (ctx.isStopping())
+                continue;
+
+            AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx();
+            AffinityTopologyVersion exchVer = ctx.cache().context().exchange().readyAffinityVersion();
+
+            if (! topVer.equals(exchVer)) {
+                info("topology version mismatch: node "  + g.name() + " " + exchVer + ", " + topVer);
+
+                GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                    @Override public boolean apply() {
+                        AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx();
+                        AffinityTopologyVersion exchVer = ctx.cache().context().exchange().readyAffinityVersion();
+
+                        return exchVer.equals(topVer);
+                    }
+                }, DFLT_TOP_WAIT_TIMEOUT);
+            }
+        }
+    }
+
     /**
      *
      */

From 8ca38cdfb6edfb84ef4e0b2c83853776d8bf3b31 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 12 Sep 2017 16:53:46 +0300
Subject: [PATCH 369/516] GG-12695: Fixed test
 IgniteCacheNearRestartRollbackSelfTest.testRestarts().

---
 .../apache/ignite/internal/IgniteKernal.java  | 66 +++++++++++--------
 ...gniteCacheNearRestartRollbackSelfTest.java |  2 -
 2 files changed, 37 insertions(+), 31 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index a1997b1adfed7..d41075f012315 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -887,37 +887,45 @@ public void start(final IgniteConfiguration cfg,
 
             ctx.add(discoMgr, false);
 
-            // Start processors before discovery manager, so they will
-            // be able to start receiving messages once discovery completes.
-            startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx));
-            startProcessor(new GridClockSyncProcessor(ctx));
-            startProcessor(new GridAffinityProcessor(ctx));
-            startProcessor(createComponent(GridSegmentationProcessor.class, ctx));
-            startProcessor(createComponent(IgniteCacheObjectProcessor.class, ctx));
-            startProcessor(new GridCacheProcessor(ctx));
-            startProcessor(new GridQueryProcessor(ctx));
-            startProcessor(new OdbcProcessor(ctx));
-            startProcessor(new GridServiceProcessor(ctx));
-            startProcessor(new GridTaskSessionProcessor(ctx));
-            startProcessor(new GridJobProcessor(ctx));
-            startProcessor(new GridTaskProcessor(ctx));
-            startProcessor((GridProcessor)SCHEDULE.createOptional(ctx));
-            startProcessor(new GridRestProcessor(ctx));
-            startProcessor(new DataStreamProcessor(ctx));
-            startProcessor((GridProcessor)IGFS.create(ctx, F.isEmpty(cfg.getFileSystemConfiguration())));
-            startProcessor(new GridContinuousProcessor(ctx));
-            startProcessor(createHadoopComponent());
-            startProcessor(new DataStructuresProcessor(ctx));
-            startProcessor(createComponent(PlatformProcessor.class, ctx));
-
-            // Start plugins.
-            for (PluginProvider provider : ctx.plugins().allProviders()) {
-                ctx.add(new GridPluginComponent(provider));
-
-                provider.start(ctx.plugins().pluginContextForProvider(provider));
+                // Start processors before discovery manager, so they will
+                // be able to start receiving messages once discovery completes.
+            try {
+                startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx));
+                startProcessor(new GridClockSyncProcessor(ctx));
+                startProcessor(new GridAffinityProcessor(ctx));
+                startProcessor(createComponent(GridSegmentationProcessor.class, ctx));
+                startProcessor(createComponent(IgniteCacheObjectProcessor.class, ctx));
+                startProcessor(new GridCacheProcessor(ctx));
+                startProcessor(new GridQueryProcessor(ctx));
+                startProcessor(new OdbcProcessor(ctx));
+                startProcessor(new GridServiceProcessor(ctx));
+                startProcessor(new GridTaskSessionProcessor(ctx));
+                startProcessor(new GridJobProcessor(ctx));
+                startProcessor(new GridTaskProcessor(ctx));
+                startProcessor((GridProcessor)SCHEDULE.createOptional(ctx));
+                startProcessor(new GridRestProcessor(ctx));
+                startProcessor(new DataStreamProcessor(ctx));
+                startProcessor((GridProcessor)IGFS.create(ctx, F.isEmpty(cfg.getFileSystemConfiguration())));
+                startProcessor(new GridContinuousProcessor(ctx));
+                startProcessor(createHadoopComponent());
+                startProcessor(new DataStructuresProcessor(ctx));
+                startProcessor(createComponent(PlatformProcessor.class, ctx));
+
+                // Start plugins.
+                for (PluginProvider provider : ctx.plugins().allProviders()) {
+                    ctx.add(new GridPluginComponent(provider));
+
+                    provider.start(ctx.plugins().pluginContextForProvider(provider));
+                }
+
+                fillNodeAttributes(clusterProc.updateNotifierEnabled());
             }
+            catch (Throwable e) {
+                // Stop discovery spi to close tcp socket.
+                ctx.discovery().stop(true);
 
-            fillNodeAttributes(clusterProc.updateNotifierEnabled());
+                throw e;
+            }
 
             gw.writeLock();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
index aea4d7782d858..3f242b5726867 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheNearRestartRollbackSelfTest.java
@@ -132,8 +132,6 @@ protected CacheConfiguration<Object, Object> cacheConfiguration(String gridName)
      */
     @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
     public void testRestarts() throws Exception {
-        fail("https://ggsystems.atlassian.net/browse/GG-12398");
-
         startGrids(4);
 
         Ignite tester = ignite(3);

From b954d26ee388416baadeeae597efd2d6060038f7 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Mon, 1 Aug 2016 10:10:51 +0300
Subject: [PATCH 370/516] IGNITE-1690: Re-enabled
 IgniteCacheCreateRestartSelfTest.testStopOriginatingNode. This closes #753.

(cherry picked from commit 0763de8)
---
 .../processors/cache/IgniteCacheCreateRestartSelfTest.java      | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheCreateRestartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheCreateRestartSelfTest.java
index e8e66c4155391..681636a34860e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheCreateRestartSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheCreateRestartSelfTest.java
@@ -71,8 +71,6 @@ public class IgniteCacheCreateRestartSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testStopOriginatingNode() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-1690");
-
         startGrids(NODES);
 
         ThreadLocalRandom rnd = ThreadLocalRandom.current();

From 88a7c0239bffcc2a3a47e67d11df360542dd0be5 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Wed, 14 Jun 2017 14:55:48 +0300
Subject: [PATCH 371/516] Fixed missing Apache header in
 HttpIgniteUpdatesChecker.

(cherry picked from commit 69aa299)
---
 .../cluster/HttpIgniteUpdatesChecker.java       | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
index c052c0997fe2a..2b93ceb1e44e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/HttpIgniteUpdatesChecker.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.ignite.internal.processors.cluster;
 
 import java.io.BufferedReader;

From 77723e433561fd908efda9abd067536494137ac4 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 13 Sep 2017 12:07:55 +0300
Subject: [PATCH 372/516] GG-12767: fixed flaky test IgniteCacheGetRestartTest.

Try avoid timeout in IgniteCacheGetRestartTest.

(cherry picked from commit b41ecd1)
---
 .../cache/distributed/IgniteCacheGetRestartTest.java  | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGetRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGetRestartTest.java
index 3b0c2fac32947..eef7750b05bf0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGetRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheGetRestartTest.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -90,6 +91,8 @@ public class IgniteCacheGetRestartTest extends GridCommonAbstractTest {
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL, "true");
+
         super.beforeTestsStarted();
 
         startGrids(SRVS);
@@ -108,11 +111,13 @@ public class IgniteCacheGetRestartTest extends GridCommonAbstractTest {
         super.afterTestsStopped();
 
         stopAllGrids();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL);
     }
 
     /** {@inheritDoc} */
     @Override protected long getTestTimeout() {
-        return TEST_TIME + 60_000;
+        return TEST_TIME + 3 * 60_000;
     }
 
     /**
@@ -198,6 +203,8 @@ private void checkRestart(final CacheConfiguration ccfg, final int restartCnt) t
                 @Override public Void call() throws Exception {
                     int nodeIdx = restartNodeIdx.getAndIncrement();
 
+                    Thread.currentThread().setName("restart-thread-" + nodeIdx);
+
                     boolean clientMode = clientNode.compareAndSet(false, true);
 
                     while (U.currentTimeMillis() < stopTime) {
@@ -219,7 +226,7 @@ private void checkRestart(final CacheConfiguration ccfg, final int restartCnt) t
 
                         IgniteInternalFuture<?> syncFut = ((IgniteCacheProxy)cache).context().preloader().syncFuture();
 
-                        while (!syncFut.isDone())
+                        while (!syncFut.isDone() && U.currentTimeMillis() < stopTime)
                             checkGet(cache);
 
                         checkGet(cache);

From 395e78cf92621df23c8173323031f655cfe53518 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 13 Apr 2017 16:36:54 +0300
Subject: [PATCH 373/516] IGNITE-3477 - Fixing flaky full API suite

(cherry picked from commit 8d2b020)
---
 .../cache/H2CacheStoreStrategy.java           | 35 +++++++++++++++----
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/H2CacheStoreStrategy.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/H2CacheStoreStrategy.java
index ccb299438ce86..6261366e10569 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/H2CacheStoreStrategy.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/H2CacheStoreStrategy.java
@@ -54,6 +54,9 @@ public class H2CacheStoreStrategy implements TestCacheStoreStrategy {
     /** Pool to get {@link Connection}s from. */
     private final JdbcConnectionPool dataSrc;
 
+    /** */
+    private final int port;
+
     /** Script that creates CACHE table. */
     private static final String CREATE_CACHE_TABLE =
         "create table if not exists CACHE(k binary not null, v binary not null, PRIMARY KEY(k));";
@@ -75,9 +78,14 @@ public class H2CacheStoreStrategy implements TestCacheStoreStrategy {
      * @throws IgniteCheckedException If failed.
      */
     public H2CacheStoreStrategy() throws IgniteCheckedException {
+        Server srv = null;
+
         try {
-            Server.createTcpServer().start();
-            dataSrc = H2CacheStoreSessionListenerFactory.createDataSource();
+            srv = Server.createTcpServer().start();
+
+            port = srv.getPort();
+
+            dataSrc = H2CacheStoreSessionListenerFactory.createDataSource(port);
 
             try (Connection conn = connection()) {
                 RunScript.execute(conn, new StringReader(CREATE_CACHE_TABLE));
@@ -86,7 +94,8 @@ public H2CacheStoreStrategy() throws IgniteCheckedException {
             }
         }
         catch (SQLException e) {
-            throw new IgniteCheckedException(e);
+            throw new IgniteCheckedException("Failed to set up cache store strategy" +
+                (srv == null ? "" : ": " + srv.getStatus()), e);
         }
     }
 
@@ -242,7 +251,7 @@ private int querySingleInt(String qry, String errorMsg) {
 
     /** {@inheritDoc} */
     @Override public void updateCacheConfiguration(CacheConfiguration<Object, Object> cfg) {
-        cfg.setCacheStoreSessionListenerFactories(new H2CacheStoreSessionListenerFactory());
+        cfg.setCacheStoreSessionListenerFactories(new H2CacheStoreSessionListenerFactory(port));
     }
 
     /** {@inheritDoc} */
@@ -260,11 +269,23 @@ public static class H2StoreFactory implements Factory<CacheStore<Object, Object>
 
     /** Serializable {@link Factory} producing H2 backed {@link CacheStoreSessionListener}s. */
     public static class H2CacheStoreSessionListenerFactory implements Factory<CacheStoreSessionListener> {
+        /** */
+        private int port;
+
+        /**
+         * @param port Port.
+         */
+        public H2CacheStoreSessionListenerFactory(int port) {
+            this.port = port;
+        }
+
         /**
          * @return Connection pool
          */
-        static JdbcConnectionPool createDataSource() {
-            JdbcConnectionPool pool = JdbcConnectionPool.create("jdbc:h2:tcp://localhost/mem:TestDb;LOCK_MODE=0", "sa", "");
+        static JdbcConnectionPool createDataSource(int port) {
+            JdbcConnectionPool pool = JdbcConnectionPool.create("jdbc:h2:tcp://localhost:" + port +
+                "/mem:TestDb;LOCK_MODE=0", "sa", "");
+
             pool.setMaxConnections(100);
             return pool;
         }
@@ -272,7 +293,7 @@ static JdbcConnectionPool createDataSource() {
         /** {@inheritDoc} */
         @Override public CacheStoreSessionListener create() {
             CacheJdbcStoreSessionListener lsnr = new CacheJdbcStoreSessionListener();
-            lsnr.setDataSource(createDataSource());
+            lsnr.setDataSource(createDataSource(port));
             return lsnr;
         }
     }

From 9ad68c735f76c6f5ce16c787e5cc07d2638d116d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 13 Apr 2017 16:36:54 +0300
Subject: [PATCH 374/516] GG-12773: Fixed potential deadlock when service is
 being deployed while node is stopping.

---
 .../service/GridServiceProcessor.java         | 171 ++++++++++--------
 .../GridServiceProcessorStopSelfTest.java     | 137 +++++++++++++-
 2 files changed, 228 insertions(+), 80 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index c452da31086d0..106aa7f678010 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -493,124 +493,137 @@ public IgniteInternalFuture<?> deployKeyAffinitySingleton(String name, Service s
     public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
         A.notNull(cfg, "cfg");
 
-        ServicesCompatibilityState state = markCompatibilityStateAsUsed();
+        if (!busyLock.enterBusy()) {
+            IgniteCheckedException e = new IgniteCheckedException("Service deployment has been cancelled (node is stopping): " +
+                cfg.getName());
 
-        validate(cfg);
-
-        ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
+            return new GridFinishedFuture<>(e);
+        }
 
-        if (!state.srvcCompatibility) {
-            Marshaller marsh = ctx.config().getMarshaller();
+        try {
 
-            LazyServiceConfiguration cfg0;
+            ServicesCompatibilityState state = markCompatibilityStateAsUsed();
 
-            try {
-                byte[] srvcBytes = U.marshal(marsh, cfg.getService());
+            validate(cfg);
 
-                cfg0 = new LazyServiceConfiguration(cfg, srvcBytes);
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to marshal service with configured marshaller [srvc=" + cfg.getService()
-                    + ", marsh=" + marsh + "]", e);
+            ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
 
-                return new GridFinishedFuture<>(e);
-            }
+            if (!state.srvcCompatibility) {
+                Marshaller marsh = ctx.config().getMarshaller();
 
-            cfg = cfg0;
-        }
+                LazyServiceConfiguration cfg0;
 
-        GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+                try {
+                    byte[] srvcBytes = U.marshal(marsh, cfg.getService());
 
-        GridServiceDeploymentFuture old = depFuts.putIfAbsent(cfg.getName(), fut);
+                    cfg0 = new LazyServiceConfiguration(cfg, srvcBytes);
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to marshal service with configured marshaller [srvc=" + cfg.getService()
+                        + ", marsh=" + marsh + "]", e);
 
-        if (old != null) {
-            if (!old.configuration().equalsIgnoreNodeFilter(cfg)) {
-                fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
-                    "different configuration) [deployed=" + old.configuration() + ", new=" + cfg + ']'));
+                    return new GridFinishedFuture<>(e);
+                }
 
-                return fut;
+                cfg = cfg0;
             }
 
-            return old;
-        }
+            GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
 
-        if (ctx.clientDisconnected()) {
-            fut.onDone(new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
-                "Failed to deploy service, client node disconnected."));
+            GridServiceDeploymentFuture old = depFuts.putIfAbsent(cfg.getName(), fut);
 
-            depFuts.remove(cfg.getName(), fut);
-        }
+            if (old != null) {
+                if (!old.configuration().equalsIgnoreNodeFilter(cfg)) {
+                    fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
+                        "different configuration) [deployed=" + old.configuration() + ", new=" + cfg + ']'));
 
-        while (true) {
-            try {
-                GridServiceDeploymentKey key = new GridServiceDeploymentKey(cfg.getName());
+                    return fut;
+                }
+
+                return old;
+            }
 
-                if (ctx.deploy().enabled())
-                    ctx.cache().context().deploy().ignoreOwnership(true);
+            if (ctx.clientDisconnected()) {
+                fut.onDone(new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
+                    "Failed to deploy service, client node disconnected."));
 
+                depFuts.remove(cfg.getName(), fut);
+            }
+
+            while (true) {
                 try {
-                    GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
-                        new GridServiceDeployment(ctx.localNodeId(), cfg));
+                    GridServiceDeploymentKey key = new GridServiceDeploymentKey(cfg.getName());
 
-                    if (dep != null) {
-                        if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
-                            // Remove future from local map.
-                            depFuts.remove(cfg.getName(), fut);
+                    if (ctx.deploy().enabled())
+                        ctx.cache().context().deploy().ignoreOwnership(true);
 
-                            fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
-                                "different configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']'));
-                        }
-                        else {
-                            Iterator<Cache.Entry<Object, Object>> it = serviceEntries(
-                                ServiceAssignmentsPredicate.INSTANCE);
+                    try {
+                        GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
+                            new GridServiceDeployment(ctx.localNodeId(), cfg));
+
+                        if (dep != null) {
+                            if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
+                                // Remove future from local map.
+                                depFuts.remove(cfg.getName(), fut);
+
+                                fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
+                                    "different configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']'));
+                            }
+                            else {
+                                Iterator<Cache.Entry<Object, Object>> it = serviceEntries(
+                                    ServiceAssignmentsPredicate.INSTANCE);
 
-                            while (it.hasNext()) {
-                                Cache.Entry<Object, Object> e = it.next();
+                                while (it.hasNext()) {
+                                    Cache.Entry<Object, Object> e = it.next();
 
-                                if (e.getKey() instanceof GridServiceAssignmentsKey) {
-                                    GridServiceAssignments assigns = (GridServiceAssignments)e.getValue();
+                                    if (e.getKey() instanceof GridServiceAssignmentsKey) {
+                                        GridServiceAssignments assigns = (GridServiceAssignments)e.getValue();
 
-                                    if (assigns.name().equals(cfg.getName())) {
-                                        // Remove future from local map.
-                                        depFuts.remove(cfg.getName(), fut);
+                                        if (assigns.name().equals(cfg.getName())) {
+                                            // Remove future from local map.
+                                            depFuts.remove(cfg.getName(), fut);
 
-                                        fut.onDone();
+                                            fut.onDone();
 
-                                        break;
+                                            break;
+                                        }
                                     }
                                 }
-                            }
 
-                            if (!dep.configuration().equalsIgnoreNodeFilter(cfg))
-                                U.warn(log, "Service already deployed with different configuration (will ignore) " +
-                                    "[deployed=" + dep.configuration() + ", new=" + cfg + ']');
+                                if (!dep.configuration().equalsIgnoreNodeFilter(cfg))
+                                    U.warn(log, "Service already deployed with different configuration (will ignore) " +
+                                        "[deployed=" + dep.configuration() + ", new=" + cfg + ']');
+                            }
                         }
                     }
-                }
-                finally {
-                    if (ctx.deploy().enabled())
-                        ctx.cache().context().deploy().ignoreOwnership(false);
-                }
+                    finally {
+                        if (ctx.deploy().enabled())
+                            ctx.cache().context().deploy().ignoreOwnership(false);
+                    }
 
-                return fut;
-            }
-            catch (ClusterTopologyCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
-            }
-            catch (IgniteCheckedException e) {
-                if (e.hasCause(ClusterTopologyCheckedException.class)) {
+                    return fut;
+                }
+                catch (ClusterTopologyCheckedException e) {
                     if (log.isDebugEnabled())
                         log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
-
-                    continue;
                 }
+                catch (IgniteCheckedException e) {
+                    if (e.hasCause(ClusterTopologyCheckedException.class)) {
+                        if (log.isDebugEnabled())
+                            log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
 
-                U.error(log, "Failed to deploy service: " + cfg.getName(), e);
+                        continue;
+                    }
 
-                return new GridFinishedFuture<>(e);
+                    U.error(log, "Failed to deploy service: " + cfg.getName(), e);
+
+                    return new GridFinishedFuture<>(e);
+                }
             }
         }
+        finally {
+            busyLock.leaveBusy();
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
index ea0ba51b60954..563c3219f28da 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
@@ -21,6 +21,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
@@ -31,8 +32,12 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.lang.GridPlainCallable;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.services.ServiceContext;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -161,8 +166,138 @@ public void testStopDuringHangedDeployment() throws Exception {
         assertTrue("Deploy future isn't completed", wait);
 
         fut.get();
+    }
 
-        Ignition.stopAll(true);
+    /**
+     * @throws Exception If fails
+     */
+    public void testServiceDeploymentCancelationOnStop() throws Exception {
+        final Ignite node = startGrid(0);
+
+        final IgniteServices services = node.services();
+        // Deploy some service.
+        services.deploy(getServiceConfiguration("myService1"));
+
+        //Stop node async, this will cancel the service #1.
+        final IgniteInternalFuture<Boolean> stopAsync = GridTestUtils.runAsync(new GridPlainCallable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                node.close();
+
+                return true;
+            }
+        }, "node-stopping-thread");
+
+        // Wait for the service #1 cancellation during node stopping.
+        // At this point node.stop process will be paused until svcCancelFinishLatch released.
+        ServiceImpl.svcCancelStartLatch.await();
+
+        final AtomicReference<IgniteFuture> queuedFuture = new AtomicReference<>();
+
+        // Try to deploy another service.
+        final IgniteInternalFuture<?> deployAsync = GridTestUtils.runAsync(new GridPlainCallable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                IgniteServices async = services.withAsync();
+
+                async.deploy(getServiceConfiguration("myService2"));
+
+                IgniteFuture<Object> future = async.future();
+
+                queuedFuture.set(future);
+
+                // Here, deployment future is added to queue and
+                // then it will be cancelled when svcCancelFinishLatch be released.
+                // So, we'll wait for queue cleaning and try to deploy one more service.
+                try {
+                    future.get();
+                }
+                catch (Exception ignore) {
+                    // Noop.
+                }
+
+                // Normally, this should fail with some Exception as node is stopping right now.
+                // But we face a deadlock here.
+                for (int i = 0; i < 5; i++) {
+                    try {
+                        services.deploy(getServiceConfiguration("service3"));
+                    }
+                    catch (Exception ignore) {
+                        // Noop.
+                    }
+                }
+
+                return true;
+            }
+        }, "svc-deploy-thread");
+
+        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return queuedFuture.get() != null;
+            }
+        }, 3000);
+
+        // Allow node to be stopped.
+        ServiceImpl.svcCancelFinishLatch.countDown();
+
+        // Wait for all service deployments have finished.
+        boolean deployDone = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                IgniteFuture fut = queuedFuture.get();
+
+                return fut != null && fut.isDone() && deployAsync.isDone();
+
+            }
+        }, 5000);
+
+        assertTrue("Node stopping and service deployment processes falls into a deadlock.", deployDone);
+
+        if (!deployDone)
+            deployAsync.cancel();
+
+        if (!stopAsync.isDone())
+            stopAsync.cancel();
+    }
+
+    /** */
+    private ServiceConfiguration getServiceConfiguration(String svcName) {
+        ServiceConfiguration svcCfg = new ServiceConfiguration();
+        svcCfg.setName(svcName);
+        svcCfg.setService(new ServiceImpl());
+        svcCfg.setTotalCount(1);
+
+        return svcCfg;
+    }
+
+    /** Dummy Implementation. */
+    static class ServiceImpl implements Service {
+        /** */
+        static final CountDownLatch svcCancelStartLatch = new CountDownLatch(1);
+
+        /** */
+        static final CountDownLatch svcCancelFinishLatch = new CountDownLatch(1);
+
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+            System.out.println("cancel service: " + ctx.executionId());
+            try {
+                svcCancelStartLatch.countDown();
+
+                svcCancelFinishLatch.await();
+            }
+            catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(ServiceContext ctx) throws Exception {
+            System.out.println("init service: " + ctx.executionId());
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+            // No-op
+        }
     }
 
     /**

From 64c337c58501f2786298cc5b92a76b8d85f6f030 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 14 Sep 2017 20:05:43 +0300
Subject: [PATCH 375/516] Fixed flaky test
 IgniteCachePeekModesAbstractTest.testNonLocalPartitionSize. Fixed wrong swap
 entries counting.

---
 .../IgniteCachePeekModesAbstractTest.java     | 37 ++++++++++++++-----
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
index fac24ccd00045..75485cdeb0d5b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
@@ -996,23 +996,40 @@ private T2<Integer, Integer> swapKeysCount(int nodeIdx) {
      * @param part Cache partition
      * @return Tuple with number of primary and backup keys (one or both will be zero).
      */
-    private T2<Integer, Integer> swapKeysCount(int nodeIdx, int part) throws IgniteCheckedException {
+    private T2<Integer, Integer> swapKeysCount(final int nodeIdx, final int part) throws IgniteCheckedException {
         GridCacheContext ctx = ((IgniteEx)ignite(nodeIdx)).context().cache().internalCache().context();
         // Swap and offheap are disabled for near cache.
         GridCacheSwapManager swapMgr = ctx.isNear() ? ctx.near().dht().context().swap() : ctx.swap();
-        //First count entries...
-        int cnt = (int)swapMgr.swapEntriesCount(part);
 
-        GridCacheAffinityManager affinity = ctx.affinity();
-        AffinityTopologyVersion topVer = affinity.affinityTopologyVersion();
+        AffinityTopologyVersion topVer = ctx.affinity().affinityTopologyVersion();
+
+        Affinity aff = ignite(nodeIdx).affinity(null);
+
+        ClusterNode node = ignite(nodeIdx).cluster().localNode();
+
+        Iterator<KeyCacheObject> it = swapMgr.swapKeyIterator(true, true, topVer);
+
+        CacheObjectContext coctx = ((IgniteEx)ignite(nodeIdx)).context().cache().internalCache()
+            .context().cacheObjectContext();
 
-        //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
-            primaryCnt = cnt;
-        else if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
-            backupCnt = cnt;
+
+        while (it.hasNext()) {
+            Integer key = it.next().value(coctx, false);
+
+            if (part != aff.partition(key))
+                continue;
+
+            if (aff.isPrimary(node, key))
+                primaryCnt++;
+            else {
+                assertTrue(aff.isBackup(node, key));
+
+                backupCnt++;
+            }
+        }
+
         return new T2<>(primaryCnt, backupCnt);
     }
 

From 46296cf8150987c40346474f352c87d1e0fb1be0 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 15 Sep 2017 11:17:53 +0700
Subject: [PATCH 376/516] IGNITE-6367 Restrict files search by IGNITE_HOME.

---
 .../visor/file/VisorLatestTextFilesTask.java  | 15 +++---
 .../visor/log/VisorLogSearchTask.java         | 28 +++++------
 .../internal/visor/util/VisorTaskUtils.java   | 47 ++++++++++++++++---
 3 files changed, 61 insertions(+), 29 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
index 55d358c78dd6e..63b526e1a1b77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
@@ -18,13 +18,11 @@
 package org.apache.ignite.internal.visor.file;
 
 import java.io.File;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.internal.visor.log.VisorLogFile;
@@ -33,6 +31,7 @@
 
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.LOG_FILES_COUNT_LIMIT;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.matchedFiles;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveIgnitePath;
 
 /**
  * Get list files matching filter.
@@ -70,13 +69,11 @@ private VisorLatestTextFilesJob(IgniteBiTuple<String, String> arg, boolean debug
             assert path != null;
             assert regexp != null;
 
-            URL url = U.resolveIgniteUrl(path);
-
-            if (url == null)
-                return null;
-
             try {
-                File folder = new File(url.toURI());
+                File folder = resolveIgnitePath(path);
+
+                if (folder == null)
+                    return null;
 
                 List<VisorLogFile> files = matchedFiles(folder, regexp);
 
@@ -98,4 +95,4 @@ private VisorLatestTextFilesJob(IgniteBiTuple<String, String> arg, boolean debug
             return S.toString(VisorLatestTextFilesJob.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
index b6552b2fca5f6..44fb0a90a5b0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
@@ -18,10 +18,8 @@
 package org.apache.ignite.internal.visor.log;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
-import java.net.URL;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -35,7 +33,6 @@
 import org.apache.ignite.internal.util.io.GridReversedLinesFileReader;
 import org.apache.ignite.internal.util.lang.GridTuple3;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorMultiNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -43,6 +40,8 @@
 
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.decode;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.matchedFiles;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveIgnitePath;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveSymbolicLink;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.textFile;
 
 /**
@@ -216,21 +215,22 @@ private List<GridTuple3<String[], Integer, Integer>> searchInFile(File f, Charse
 
         /** {@inheritDoc} */
         @Override protected Collection<VisorLogSearchResult> run(VisorLogSearchArg arg) {
-            URL url = U.resolveIgniteUrl(arg.folder);
+            try {
+                File folder = resolveIgnitePath(arg.folder);
 
-            if (url == null)
-                throw new IgniteException(new FileNotFoundException("Log folder not found: " + arg.folder));
+                if (folder == null)
+                    return null;
 
-            UUID uuid = ignite.localNode().id();
-            String nid = uuid.toString().toLowerCase();
+                folder = resolveSymbolicLink(folder);
 
-            String filePtrn = arg.filePtrn.replace("@nid8", nid.substring(0, 8)).replace("@nid", nid);
+                UUID uuid = ignite.localNode().id();
+                String nid = uuid.toString().toLowerCase();
 
-            try {
-                File fld = new File(url.toURI());
-                int pathIdx = (fld.isDirectory() ? fld : fld.getParentFile()).getAbsolutePath().length() + 1;
+                String filePtrn = arg.filePtrn.replace("@nid8", nid.substring(0, 8)).replace("@nid", nid);
+
+                int pathIdx = (folder.isDirectory() ? folder : folder.getParentFile()).getAbsolutePath().length() + 1;
 
-                List<VisorLogFile> matchingFiles = matchedFiles(fld, filePtrn);
+                List<VisorLogFile> matchingFiles = matchedFiles(folder, filePtrn);
 
                 Collection<VisorLogSearchResult> results = new ArrayList<>(arg.limit);
 
@@ -276,4 +276,4 @@ private List<GridTuple3<String[], Integer, Integer>> searchInFile(File f, Charse
             return S.toString(VisorLogSearchJob.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
index 3f5003a03d401..7518d93e1aa8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
@@ -30,7 +30,10 @@
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -494,6 +497,34 @@ public static Collection<VisorGridEvent> collectEvents(Ignite ignite, String evt
         return res;
     }
 
+    /**
+     * @param path Path to resolve only relative to IGNITE_HOME.
+     * @return Resolved path as file, or {@code null} if path cannot be resolved.
+     * @throws IOException If failed to resolve path.
+     */
+    public static File resolveIgnitePath(String path) throws IOException {
+        File folder = U.resolveIgnitePath(path);
+
+        if (folder == null)
+            return null;
+
+        if (!folder.toPath().toRealPath(LinkOption.NOFOLLOW_LINKS).startsWith(Paths.get(U.getIgniteHome())))
+            return null;
+
+        return folder;
+    }
+
+    /**
+     * @param file File to resolve.
+     * @return Resolved file if it is a symbolic link or original file.
+     * @throws IOException If failed to resolve symlink.
+     */
+    public static File resolveSymbolicLink(File file) throws IOException {
+        Path path = file.toPath();
+
+        return Files.isSymbolicLink(path) ? Files.readSymbolicLink(path).toFile() : file;
+    }
+
     /**
      * Finds all files in folder and in it's sub-tree of specified depth.
      *
@@ -501,8 +532,11 @@ public static Collection<VisorGridEvent> collectEvents(Ignite ignite, String evt
      * @param maxDepth Depth of the tree. If 1 - just look in the folder, no sub-folders.
      * @param filter file filter.
      * @return List of found files.
+     * @throws IOException If failed to list files.
      */
-    public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) {
+    public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) throws IOException {
+        file = resolveSymbolicLink(file);
+
         if (file.isDirectory()) {
             File[] files = (filter == null) ? file.listFiles() : file.listFiles(filter);
 
@@ -525,12 +559,13 @@ else if (maxDepth > 1)
     }
 
     /**
-     * @param fld Folder with files to match.
+     * @param file Folder with files to match.
      * @param ptrn Pattern to match against file name.
      * @return Collection of matched files.
+     * @throws IOException If failed to filter files.
      */
-    public static List<VisorLogFile> matchedFiles(File fld, final String ptrn) {
-        List<VisorLogFile> files = fileTree(fld, MAX_FOLDER_DEPTH,
+    public static List<VisorLogFile> matchedFiles(File file, final String ptrn) throws IOException {
+        List<VisorLogFile> files = fileTree(file, MAX_FOLDER_DEPTH,
             new FileFilter() {
                 @Override public boolean accept(File f) {
                     return !f.isHidden() && (f.isDirectory() || f.isFile() && f.getName().matches(ptrn));
@@ -873,8 +908,6 @@ public static List<Process> startLocalNode(@Nullable IgniteLogger log, String cf
         if (cmdFilePath == null || !cmdFilePath.exists())
             throw new FileNotFoundException(String.format("File not found: %s", cmdFile));
 
-        String ignite = cmdFilePath.getCanonicalPath();
-
         File nodesCfgPath = U.resolveIgnitePath(cfgPath);
 
         if (nodesCfgPath == null || !nodesCfgPath.exists())
@@ -887,6 +920,8 @@ public static List<Process> startLocalNode(@Nullable IgniteLogger log, String cf
         List<Process> run = new ArrayList<>();
 
         try {
+            String ignite = cmdFilePath.getCanonicalPath();
+
             for (int i = 0; i < nodesToStart; i++) {
                 if (U.isMacOs()) {
                     Map<String, String> macEnv = new HashMap<>(System.getenv());

From 86cb71985b6f6407264b3dbd40d2feed4d446a79 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 15 Sep 2017 11:17:53 +0700
Subject: [PATCH 377/516] IGNITE-6367 Restrict files search by IGNITE_HOME.
 (cherry picked from commit 46296cf)

---
 .../visor/file/VisorLatestTextFilesTask.java  | 15 +++---
 .../visor/log/VisorLogSearchTask.java         | 28 +++++------
 .../internal/visor/util/VisorTaskUtils.java   | 47 ++++++++++++++++---
 3 files changed, 61 insertions(+), 29 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
index 55d358c78dd6e..63b526e1a1b77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
@@ -18,13 +18,11 @@
 package org.apache.ignite.internal.visor.file;
 
 import java.io.File;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.internal.visor.log.VisorLogFile;
@@ -33,6 +31,7 @@
 
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.LOG_FILES_COUNT_LIMIT;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.matchedFiles;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveIgnitePath;
 
 /**
  * Get list files matching filter.
@@ -70,13 +69,11 @@ private VisorLatestTextFilesJob(IgniteBiTuple<String, String> arg, boolean debug
             assert path != null;
             assert regexp != null;
 
-            URL url = U.resolveIgniteUrl(path);
-
-            if (url == null)
-                return null;
-
             try {
-                File folder = new File(url.toURI());
+                File folder = resolveIgnitePath(path);
+
+                if (folder == null)
+                    return null;
 
                 List<VisorLogFile> files = matchedFiles(folder, regexp);
 
@@ -98,4 +95,4 @@ private VisorLatestTextFilesJob(IgniteBiTuple<String, String> arg, boolean debug
             return S.toString(VisorLatestTextFilesJob.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
index b6552b2fca5f6..44fb0a90a5b0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
@@ -18,10 +18,8 @@
 package org.apache.ignite.internal.visor.log;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
-import java.net.URL;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -35,7 +33,6 @@
 import org.apache.ignite.internal.util.io.GridReversedLinesFileReader;
 import org.apache.ignite.internal.util.lang.GridTuple3;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorMultiNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -43,6 +40,8 @@
 
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.decode;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.matchedFiles;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveIgnitePath;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveSymbolicLink;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.textFile;
 
 /**
@@ -216,21 +215,22 @@ private List<GridTuple3<String[], Integer, Integer>> searchInFile(File f, Charse
 
         /** {@inheritDoc} */
         @Override protected Collection<VisorLogSearchResult> run(VisorLogSearchArg arg) {
-            URL url = U.resolveIgniteUrl(arg.folder);
+            try {
+                File folder = resolveIgnitePath(arg.folder);
 
-            if (url == null)
-                throw new IgniteException(new FileNotFoundException("Log folder not found: " + arg.folder));
+                if (folder == null)
+                    return null;
 
-            UUID uuid = ignite.localNode().id();
-            String nid = uuid.toString().toLowerCase();
+                folder = resolveSymbolicLink(folder);
 
-            String filePtrn = arg.filePtrn.replace("@nid8", nid.substring(0, 8)).replace("@nid", nid);
+                UUID uuid = ignite.localNode().id();
+                String nid = uuid.toString().toLowerCase();
 
-            try {
-                File fld = new File(url.toURI());
-                int pathIdx = (fld.isDirectory() ? fld : fld.getParentFile()).getAbsolutePath().length() + 1;
+                String filePtrn = arg.filePtrn.replace("@nid8", nid.substring(0, 8)).replace("@nid", nid);
+
+                int pathIdx = (folder.isDirectory() ? folder : folder.getParentFile()).getAbsolutePath().length() + 1;
 
-                List<VisorLogFile> matchingFiles = matchedFiles(fld, filePtrn);
+                List<VisorLogFile> matchingFiles = matchedFiles(folder, filePtrn);
 
                 Collection<VisorLogSearchResult> results = new ArrayList<>(arg.limit);
 
@@ -276,4 +276,4 @@ private List<GridTuple3<String[], Integer, Integer>> searchInFile(File f, Charse
             return S.toString(VisorLogSearchJob.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
index 3f5003a03d401..7518d93e1aa8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
@@ -30,7 +30,10 @@
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -494,6 +497,34 @@ public static Collection<VisorGridEvent> collectEvents(Ignite ignite, String evt
         return res;
     }
 
+    /**
+     * @param path Path to resolve only relative to IGNITE_HOME.
+     * @return Resolved path as file, or {@code null} if path cannot be resolved.
+     * @throws IOException If failed to resolve path.
+     */
+    public static File resolveIgnitePath(String path) throws IOException {
+        File folder = U.resolveIgnitePath(path);
+
+        if (folder == null)
+            return null;
+
+        if (!folder.toPath().toRealPath(LinkOption.NOFOLLOW_LINKS).startsWith(Paths.get(U.getIgniteHome())))
+            return null;
+
+        return folder;
+    }
+
+    /**
+     * @param file File to resolve.
+     * @return Resolved file if it is a symbolic link or original file.
+     * @throws IOException If failed to resolve symlink.
+     */
+    public static File resolveSymbolicLink(File file) throws IOException {
+        Path path = file.toPath();
+
+        return Files.isSymbolicLink(path) ? Files.readSymbolicLink(path).toFile() : file;
+    }
+
     /**
      * Finds all files in folder and in it's sub-tree of specified depth.
      *
@@ -501,8 +532,11 @@ public static Collection<VisorGridEvent> collectEvents(Ignite ignite, String evt
      * @param maxDepth Depth of the tree. If 1 - just look in the folder, no sub-folders.
      * @param filter file filter.
      * @return List of found files.
+     * @throws IOException If failed to list files.
      */
-    public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) {
+    public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) throws IOException {
+        file = resolveSymbolicLink(file);
+
         if (file.isDirectory()) {
             File[] files = (filter == null) ? file.listFiles() : file.listFiles(filter);
 
@@ -525,12 +559,13 @@ else if (maxDepth > 1)
     }
 
     /**
-     * @param fld Folder with files to match.
+     * @param file Folder with files to match.
      * @param ptrn Pattern to match against file name.
      * @return Collection of matched files.
+     * @throws IOException If failed to filter files.
      */
-    public static List<VisorLogFile> matchedFiles(File fld, final String ptrn) {
-        List<VisorLogFile> files = fileTree(fld, MAX_FOLDER_DEPTH,
+    public static List<VisorLogFile> matchedFiles(File file, final String ptrn) throws IOException {
+        List<VisorLogFile> files = fileTree(file, MAX_FOLDER_DEPTH,
             new FileFilter() {
                 @Override public boolean accept(File f) {
                     return !f.isHidden() && (f.isDirectory() || f.isFile() && f.getName().matches(ptrn));
@@ -873,8 +908,6 @@ public static List<Process> startLocalNode(@Nullable IgniteLogger log, String cf
         if (cmdFilePath == null || !cmdFilePath.exists())
             throw new FileNotFoundException(String.format("File not found: %s", cmdFile));
 
-        String ignite = cmdFilePath.getCanonicalPath();
-
         File nodesCfgPath = U.resolveIgnitePath(cfgPath);
 
         if (nodesCfgPath == null || !nodesCfgPath.exists())
@@ -887,6 +920,8 @@ public static List<Process> startLocalNode(@Nullable IgniteLogger log, String cf
         List<Process> run = new ArrayList<>();
 
         try {
+            String ignite = cmdFilePath.getCanonicalPath();
+
             for (int i = 0; i < nodesToStart; i++) {
                 if (U.isMacOs()) {
                     Map<String, String> macEnv = new HashMap<>(System.getenv());

From 1d165d9c07fc8e13d6ee8d4e7b9cf1a83144e75e Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 15 Sep 2017 11:17:53 +0700
Subject: [PATCH 378/516] IGNITE-6367 Restrict files search by IGNITE_HOME.
 (cherry picked from commit 46296cf)

---
 .../visor/file/VisorLatestTextFilesTask.java  | 15 +++---
 .../visor/log/VisorLogSearchTask.java         | 28 +++++------
 .../internal/visor/util/VisorTaskUtils.java   | 47 ++++++++++++++++---
 3 files changed, 61 insertions(+), 29 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
index 55d358c78dd6e..63b526e1a1b77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorLatestTextFilesTask.java
@@ -18,13 +18,11 @@
 package org.apache.ignite.internal.visor.file;
 
 import java.io.File;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.internal.visor.log.VisorLogFile;
@@ -33,6 +31,7 @@
 
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.LOG_FILES_COUNT_LIMIT;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.matchedFiles;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveIgnitePath;
 
 /**
  * Get list files matching filter.
@@ -70,13 +69,11 @@ private VisorLatestTextFilesJob(IgniteBiTuple<String, String> arg, boolean debug
             assert path != null;
             assert regexp != null;
 
-            URL url = U.resolveIgniteUrl(path);
-
-            if (url == null)
-                return null;
-
             try {
-                File folder = new File(url.toURI());
+                File folder = resolveIgnitePath(path);
+
+                if (folder == null)
+                    return null;
 
                 List<VisorLogFile> files = matchedFiles(folder, regexp);
 
@@ -98,4 +95,4 @@ private VisorLatestTextFilesJob(IgniteBiTuple<String, String> arg, boolean debug
             return S.toString(VisorLatestTextFilesJob.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
index b6552b2fca5f6..44fb0a90a5b0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java
@@ -18,10 +18,8 @@
 package org.apache.ignite.internal.visor.log;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
-import java.net.URL;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -35,7 +33,6 @@
 import org.apache.ignite.internal.util.io.GridReversedLinesFileReader;
 import org.apache.ignite.internal.util.lang.GridTuple3;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorMultiNodeTask;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -43,6 +40,8 @@
 
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.decode;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.matchedFiles;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveIgnitePath;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.resolveSymbolicLink;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.textFile;
 
 /**
@@ -216,21 +215,22 @@ private List<GridTuple3<String[], Integer, Integer>> searchInFile(File f, Charse
 
         /** {@inheritDoc} */
         @Override protected Collection<VisorLogSearchResult> run(VisorLogSearchArg arg) {
-            URL url = U.resolveIgniteUrl(arg.folder);
+            try {
+                File folder = resolveIgnitePath(arg.folder);
 
-            if (url == null)
-                throw new IgniteException(new FileNotFoundException("Log folder not found: " + arg.folder));
+                if (folder == null)
+                    return null;
 
-            UUID uuid = ignite.localNode().id();
-            String nid = uuid.toString().toLowerCase();
+                folder = resolveSymbolicLink(folder);
 
-            String filePtrn = arg.filePtrn.replace("@nid8", nid.substring(0, 8)).replace("@nid", nid);
+                UUID uuid = ignite.localNode().id();
+                String nid = uuid.toString().toLowerCase();
 
-            try {
-                File fld = new File(url.toURI());
-                int pathIdx = (fld.isDirectory() ? fld : fld.getParentFile()).getAbsolutePath().length() + 1;
+                String filePtrn = arg.filePtrn.replace("@nid8", nid.substring(0, 8)).replace("@nid", nid);
+
+                int pathIdx = (folder.isDirectory() ? folder : folder.getParentFile()).getAbsolutePath().length() + 1;
 
-                List<VisorLogFile> matchingFiles = matchedFiles(fld, filePtrn);
+                List<VisorLogFile> matchingFiles = matchedFiles(folder, filePtrn);
 
                 Collection<VisorLogSearchResult> results = new ArrayList<>(arg.limit);
 
@@ -276,4 +276,4 @@ private List<GridTuple3<String[], Integer, Integer>> searchInFile(File f, Charse
             return S.toString(VisorLogSearchJob.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
index 3f5003a03d401..7518d93e1aa8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
@@ -30,7 +30,10 @@
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -494,6 +497,34 @@ public static Collection<VisorGridEvent> collectEvents(Ignite ignite, String evt
         return res;
     }
 
+    /**
+     * @param path Path to resolve only relative to IGNITE_HOME.
+     * @return Resolved path as file, or {@code null} if path cannot be resolved.
+     * @throws IOException If failed to resolve path.
+     */
+    public static File resolveIgnitePath(String path) throws IOException {
+        File folder = U.resolveIgnitePath(path);
+
+        if (folder == null)
+            return null;
+
+        if (!folder.toPath().toRealPath(LinkOption.NOFOLLOW_LINKS).startsWith(Paths.get(U.getIgniteHome())))
+            return null;
+
+        return folder;
+    }
+
+    /**
+     * @param file File to resolve.
+     * @return Resolved file if it is a symbolic link or original file.
+     * @throws IOException If failed to resolve symlink.
+     */
+    public static File resolveSymbolicLink(File file) throws IOException {
+        Path path = file.toPath();
+
+        return Files.isSymbolicLink(path) ? Files.readSymbolicLink(path).toFile() : file;
+    }
+
     /**
      * Finds all files in folder and in it's sub-tree of specified depth.
      *
@@ -501,8 +532,11 @@ public static Collection<VisorGridEvent> collectEvents(Ignite ignite, String evt
      * @param maxDepth Depth of the tree. If 1 - just look in the folder, no sub-folders.
      * @param filter file filter.
      * @return List of found files.
+     * @throws IOException If failed to list files.
      */
-    public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) {
+    public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) throws IOException {
+        file = resolveSymbolicLink(file);
+
         if (file.isDirectory()) {
             File[] files = (filter == null) ? file.listFiles() : file.listFiles(filter);
 
@@ -525,12 +559,13 @@ else if (maxDepth > 1)
     }
 
     /**
-     * @param fld Folder with files to match.
+     * @param file Folder with files to match.
      * @param ptrn Pattern to match against file name.
      * @return Collection of matched files.
+     * @throws IOException If failed to filter files.
      */
-    public static List<VisorLogFile> matchedFiles(File fld, final String ptrn) {
-        List<VisorLogFile> files = fileTree(fld, MAX_FOLDER_DEPTH,
+    public static List<VisorLogFile> matchedFiles(File file, final String ptrn) throws IOException {
+        List<VisorLogFile> files = fileTree(file, MAX_FOLDER_DEPTH,
             new FileFilter() {
                 @Override public boolean accept(File f) {
                     return !f.isHidden() && (f.isDirectory() || f.isFile() && f.getName().matches(ptrn));
@@ -873,8 +908,6 @@ public static List<Process> startLocalNode(@Nullable IgniteLogger log, String cf
         if (cmdFilePath == null || !cmdFilePath.exists())
             throw new FileNotFoundException(String.format("File not found: %s", cmdFile));
 
-        String ignite = cmdFilePath.getCanonicalPath();
-
         File nodesCfgPath = U.resolveIgnitePath(cfgPath);
 
         if (nodesCfgPath == null || !nodesCfgPath.exists())
@@ -887,6 +920,8 @@ public static List<Process> startLocalNode(@Nullable IgniteLogger log, String cf
         List<Process> run = new ArrayList<>();
 
         try {
+            String ignite = cmdFilePath.getCanonicalPath();
+
             for (int i = 0; i < nodesToStart; i++) {
                 if (U.isMacOs()) {
                     Map<String, String> macEnv = new HashMap<>(System.getenv());

From 6dfdaedc0b3cb9c7ea617d78753859d0e8b77595 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 14 Sep 2017 20:05:43 +0300
Subject: [PATCH 379/516] GG-12692: Fixed flaky test
 IgniteCachePeekModesAbstractTest.testNonLocalPartitionSize.

(cherry picked from commit 64c337c)
---
 .../IgniteCachePeekModesAbstractTest.java     | 37 ++++++++++++++-----
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
index fac24ccd00045..75485cdeb0d5b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCachePeekModesAbstractTest.java
@@ -996,23 +996,40 @@ private T2<Integer, Integer> swapKeysCount(int nodeIdx) {
      * @param part Cache partition
      * @return Tuple with number of primary and backup keys (one or both will be zero).
      */
-    private T2<Integer, Integer> swapKeysCount(int nodeIdx, int part) throws IgniteCheckedException {
+    private T2<Integer, Integer> swapKeysCount(final int nodeIdx, final int part) throws IgniteCheckedException {
         GridCacheContext ctx = ((IgniteEx)ignite(nodeIdx)).context().cache().internalCache().context();
         // Swap and offheap are disabled for near cache.
         GridCacheSwapManager swapMgr = ctx.isNear() ? ctx.near().dht().context().swap() : ctx.swap();
-        //First count entries...
-        int cnt = (int)swapMgr.swapEntriesCount(part);
 
-        GridCacheAffinityManager affinity = ctx.affinity();
-        AffinityTopologyVersion topVer = affinity.affinityTopologyVersion();
+        AffinityTopologyVersion topVer = ctx.affinity().affinityTopologyVersion();
+
+        Affinity aff = ignite(nodeIdx).affinity(null);
+
+        ClusterNode node = ignite(nodeIdx).cluster().localNode();
+
+        Iterator<KeyCacheObject> it = swapMgr.swapKeyIterator(true, true, topVer);
+
+        CacheObjectContext coctx = ((IgniteEx)ignite(nodeIdx)).context().cache().internalCache()
+            .context().cacheObjectContext();
 
-        //And then find out whether they are primary or backup ones.
         int primaryCnt = 0;
         int backupCnt = 0;
-        if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
-            primaryCnt = cnt;
-        else if (affinity.primaryByPartition(ctx.localNode(), part, topVer))
-            backupCnt = cnt;
+
+        while (it.hasNext()) {
+            Integer key = it.next().value(coctx, false);
+
+            if (part != aff.partition(key))
+                continue;
+
+            if (aff.isPrimary(node, key))
+                primaryCnt++;
+            else {
+                assertTrue(aff.isBackup(node, key));
+
+                backupCnt++;
+            }
+        }
+
         return new T2<>(primaryCnt, backupCnt);
     }
 

From a85a42d495bf449b7786fd9a8362be54b1fd2eea Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dgovorukhin@gridgain.com>
Date: Mon, 23 Jan 2017 20:35:02 +0300
Subject: [PATCH 380/516] Bakport fix for flacky test
 GridServiceProcessorProxySelfTest.testLocalProxyInvocation.

(cherry picked from commit 3587ac6)
---
 .../GridServiceProcessorProxySelfTest.java    | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
index 850cb5702b5b0..1418973940b41 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
@@ -20,9 +20,11 @@
 import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceContext;
@@ -240,12 +242,27 @@ public void testLocalProxyInvocation() throws Exception {
         ignite.services().deployNodeSingleton(name, new MapServiceImpl<String, Integer>());
 
         for (int i = 0; i < nodeCount(); i++) {
-            MapService<Integer, String> svc =  grid(i).services().serviceProxy(name, MapService.class, false);
+            final int idx = i;
+
+            final AtomicReference< MapService<Integer, String>> ref = new AtomicReference<>();
+
+            //wait because after deployNodeSingleton we don't have guarantees what service was deploy.
+            boolean wait = GridTestUtils.waitForCondition(new PA() {
+                @Override public boolean apply() {
+                    MapService<Integer, String> svc = grid(idx)
+                        .services()
+                        .serviceProxy(name, MapService.class, false);
+
+                    ref.set(svc);
+
+                    return svc instanceof Service;
+                }
+            }, 2000);
 
             // Make sure service is a local instance.
-            assertTrue("Invalid service instance [srv=" + svc + ", node=" + i + ']', svc instanceof Service);
+            assertTrue("Invalid service instance [srv=" + ref.get() + ", node=" + i + ']', wait);
 
-            svc.put(i, Integer.toString(i));
+            ref.get().put(i, Integer.toString(i));
         }
 
         MapService<Integer, String> map = ignite.services().serviceProxy(name, MapService.class, false);

From 238c7723e95dfed8bbd69bf26c98011490780a68 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 18 Sep 2017 12:54:33 +0300
Subject: [PATCH 381/516] GG-12717: Backport fix CacheJdbcPojoStoreTest from
 master.

---
 .../store/jdbc/CacheAbstractJdbcStore.java    |  67 +++++----
 .../cache/store/CacheOsStoreManager.java      |   3 +
 .../store/GridCacheStoreManagerAdapter.java   |   8 ++
 .../CacheJdbcPojoStoreAbstractSelfTest.java   |  19 ++-
 ...naryMarshallerStoreKeepBinarySelfTest.java |  28 ++++
 ...rStoreKeepBinaryWithSqlEscapeSelfTest.java |  28 ++++
 .../store/jdbc/CacheJdbcPojoStoreTest.java    | 136 ++++++++++++++----
 .../testframework/junits/IgniteMock.java      |   4 +
 .../testsuites/IgniteCacheTestSuite.java      |   4 +
 9 files changed, 237 insertions(+), 60 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
index 817b1a5da8531..64bdec90f5296 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java
@@ -61,6 +61,7 @@
 import org.apache.ignite.cache.store.jdbc.dialect.OracleDialect;
 import org.apache.ignite.cache.store.jdbc.dialect.SQLServerDialect;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.binary.BinaryEnumObjectImpl;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
@@ -560,22 +561,35 @@ private JdbcTypeField[] translateFields(Collection<CacheTypeFieldMetadata> oldFl
 
     /**
      * @param type Type name to check.
+     * @param binarySupported True if binary marshaller enable.
      * @return {@code True} if class not found.
      */
-    protected TypeKind kindForName(String type) {
+    protected TypeKind kindForName(String type, boolean binarySupported) {
         if (BUILT_IN_TYPES.contains(type))
             return TypeKind.BUILT_IN;
 
+        if (binarySupported)
+            return TypeKind.BINARY;
+
         try {
             Class.forName(type);
 
             return TypeKind.POJO;
         }
-        catch(ClassNotFoundException ignored) {
-            return TypeKind.BINARY;
+        catch (ClassNotFoundException e) {
+            throw new CacheException("Failed to find class " + type +
+                " (make sure the class is present in classPath or use BinaryMarshaller)", e);
         }
     }
 
+    /**
+     * @param type Type name to check.
+     * @return {@code True} if class not found.
+     */
+    protected TypeKind kindForName(String type) {
+        return kindForName(type, ignite.configuration().getMarshaller() instanceof BinaryMarshaller);
+    }
+
     /**
      * @param cacheName Cache name to check mappings for.
      * @return Type mappings for specified cache name.
@@ -640,11 +654,7 @@ private Map<Object, EntryMapping> getOrCreateCacheMappings(@Nullable String cach
                     String keyType = type.getKeyType();
                     String valType = type.getValueType();
 
-                    TypeKind keyKind = kindForName(keyType);
-
-                    if (!binarySupported && keyKind == TypeKind.BINARY)
-                        throw new CacheException("Key type has no class [cache=" + U.maskName(cacheName) +
-                            ", type=" + keyType + "]");
+                    TypeKind keyKind = kindForName(keyType, binarySupported);
 
                     checkTypeConfiguration(cacheName, keyKind, keyType, type.getKeyFields());
 
@@ -654,21 +664,11 @@ private Map<Object, EntryMapping> getOrCreateCacheMappings(@Nullable String cach
                         throw new CacheException("Key type must be unique in type metadata [cache=" +
                             U.maskName(cacheName) + ", type=" + keyType + "]");
 
-                    TypeKind valKind = kindForName(valType);
+                    TypeKind valKind = kindForName(valType, binarySupported);
 
                     checkTypeConfiguration(cacheName, valKind, valType, type.getValueFields());
 
                     entryMappings.put(keyTypeId, new EntryMapping(cacheName, dialect, type, keyKind, valKind, sqlEscapeAll));
-
-                    // Add one more binding to binary typeId for POJOs,
-                    // because object could be passed to store in binary format.
-                    if (binarySupported && keyKind == TypeKind.POJO) {
-                        keyTypeId = typeIdForTypeName(TypeKind.BINARY, keyType);
-
-                        valKind = valKind == TypeKind.POJO ? TypeKind.BINARY : valKind;
-
-                        entryMappings.put(keyTypeId, new EntryMapping(cacheName, dialect, type, TypeKind.BINARY, valKind, sqlEscapeAll));
-                    }
                 }
 
                 Map<String, Map<Object, EntryMapping>> mappings = new HashMap<>(cacheMappings);
@@ -1419,10 +1419,17 @@ protected void fillParameter(PreparedStatement stmt, int idx, JdbcTypeField fiel
                             // No-op.
                     }
                 }
-                else if (field.getJavaFieldType().isEnum() && fieldVal instanceof Enum) {
-                    Enum val = (Enum)fieldVal;
-                    
-                    fieldVal = NUMERIC_TYPES.contains(field.getDatabaseFieldType()) ? val.ordinal() : val.name();
+                else if (field.getJavaFieldType().isEnum()) {
+                    if (fieldVal instanceof Enum) {
+                        Enum val = (Enum)fieldVal;
+
+                        fieldVal = NUMERIC_TYPES.contains(field.getDatabaseFieldType()) ? val.ordinal() : val.name();
+                    }
+                    else if (fieldVal instanceof BinaryEnumObjectImpl) {
+                        BinaryEnumObjectImpl val = (BinaryEnumObjectImpl)fieldVal;
+
+                        fieldVal = val.enumOrdinal();
+                    }
                 }
 
                 stmt.setObject(idx, fieldVal);
@@ -1474,14 +1481,14 @@ protected int fillKeyParameters(PreparedStatement stmt, EntryMapping m, Object k
      */
     protected int fillValueParameters(PreparedStatement stmt, int idx, EntryMapping em, Object val)
         throws CacheWriterException {
-        TypeKind valKind = em.valueKind();
-
-        // Object could be passed by cache in binary format in case of cache configured with setStoreKeepBinary(true).
-        if (valKind == TypeKind.POJO && val instanceof BinaryObject)
-            valKind = TypeKind.BINARY;
-
         for (JdbcTypeField field : em.uniqValFlds) {
-            Object fieldVal = extractParameter(em.cacheName, em.valueType(), valKind, field.getJavaFieldName(), val);
+            Object fieldVal = extractParameter(
+                em.cacheName,
+                em.valueType(),
+                em.valueKind(),
+                field.getJavaFieldName(),
+                val
+            );
 
             fillParameter(stmt, idx++, field, fieldVal);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java
index 27771ff3e629a..2e23d04f735d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheOsStoreManager.java
@@ -77,6 +77,9 @@ public CacheOsStoreManager(GridKernalContext ctx, CacheConfiguration cfg) {
 
     /** {@inheritDoc} */
     @Override public boolean convertBinary() {
+        if (alwaysKeepBinary)
+            return false;
+
         return configuredConvertBinary() && !(cfgStore instanceof PlatformCacheStore);
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 6ab8c56578593..5d5c7d667c94e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -33,9 +33,11 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cache.store.CacheStoreSession;
 import org.apache.ignite.cache.store.CacheStoreSessionListener;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
@@ -113,6 +115,9 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
     /** */
     private boolean globalSesLsnrs;
 
+    /** Always keep binary. */
+    protected boolean alwaysKeepBinary;
+
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public void initialize(@Nullable CacheStore cfgStore, Map sesHolders) throws IgniteCheckedException {
@@ -148,6 +153,9 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
         sesHolder = sesHolder0;
 
         locStore = U.hasAnnotation(cfgStore, CacheLocalStore.class);
+
+        if (cfgStore instanceof CacheJdbcPojoStore)
+            alwaysKeepBinary = true;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
index 9e59769503f53..5ca6ca45364f8 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ConnectorConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -221,6 +222,7 @@ protected CacheConfiguration cacheConfiguration() throws Exception {
         cc.setAtomicityMode(transactional ? TRANSACTIONAL : ATOMIC);
         cc.setSwapEnabled(false);
         cc.setWriteBehindEnabled(false);
+        cc.setStoreKeepBinary(storeKeepBinary());
 
         CacheJdbcPojoStoreFactory<Object, Object> storeFactory = new CacheJdbcPojoStoreFactory<>();
         storeFactory.setDialect(new H2Dialect());
@@ -237,6 +239,13 @@ protected CacheConfiguration cacheConfiguration() throws Exception {
         return cc;
     }
 
+    /**
+     * @return Flag indicate keep value in binary format or not.
+     */
+    protected boolean storeKeepBinary(){
+        return false;
+    }
+
     /**
      * Fill in-memory database with sample data.
      *
@@ -446,6 +455,8 @@ public void testLoadCachePrimitiveKeysTx() throws Exception {
      * @throws Exception If failed.
      */
     private void checkPutRemove() throws Exception {
+        boolean binaryMarshaller = marshaller() instanceof BinaryMarshaller || marshaller() == null;
+
         IgniteCache<Object, Person> c1 = grid().cache(CACHE_NAME);
 
         Connection conn = getConnection();
@@ -480,7 +491,9 @@ private void checkPutRemove() throws Exception {
             assertEquals(-2, rs.getInt(2));
             assertEquals(testDate, rs.getDate(3));
             assertEquals("Person-to-test-put-insert", rs.getString(4));
-            assertEquals(testGender.toString(), rs.getString(5));
+
+            assertEquals(testGender.toString(),
+                binaryMarshaller ? Gender.values()[rs.getInt(5)].toString(): rs.getString(5));
 
             assertFalse("Unexpected more data in result set", rs.next());
 
@@ -499,7 +512,9 @@ private void checkPutRemove() throws Exception {
             assertEquals(-3, rs.getInt(2));
             assertEquals(testDate, rs.getDate(3));
             assertEquals("Person-to-test-put-update", rs.getString(4));
-            assertEquals(testGender.toString(), rs.getString(5));
+
+            assertEquals(testGender.toString(),
+                binaryMarshaller ? Gender.values()[rs.getInt(5)].toString(): rs.getString(5));
 
             assertFalse("Unexpected more data in result set", rs.next());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java
new file mode 100644
index 0000000000000..dfca8647e2840
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.jdbc;
+
+/**
+ *
+ */
+public class CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest extends CacheJdbcPojoStoreBinaryMarshallerSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean storeKeepBinary() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java
new file mode 100644
index 0000000000000..c7e1f794ab208
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.jdbc;
+
+/**
+ *
+ */
+public class CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest extends CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean storeKeepBinary() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
index 849cab79a03be..f17ece4e59686 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.cache.store.jdbc;
 
+import java.lang.reflect.Field;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
@@ -29,18 +30,22 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import javax.cache.integration.CacheWriterException;
-
+import org.apache.ignite.Ignite;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
 import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect;
 import org.apache.ignite.cache.store.jdbc.model.Organization;
 import org.apache.ignite.cache.store.jdbc.model.OrganizationKey;
 import org.apache.ignite.cache.store.jdbc.model.Person;
 import org.apache.ignite.cache.store.jdbc.model.PersonComplexKey;
 import org.apache.ignite.cache.store.jdbc.model.PersonKey;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.config.GridTestProperties;
 import org.apache.ignite.testframework.junits.cache.GridAbstractCacheStoreSelfTest;
 import org.h2.jdbcx.JdbcConnectionPool;
 
@@ -57,6 +62,12 @@ public class CacheJdbcPojoStoreTest extends GridAbstractCacheStoreSelfTest<Cache
     /** Person count. */
     protected static final int PERSON_CNT = 100000;
 
+    /** Ignite. */
+    private Ignite ig;
+
+    /** Binary enable. */
+    private boolean binaryEnable;
+
     /**
      * @throws Exception If failed.
      */
@@ -226,6 +237,12 @@ public CacheJdbcPojoStoreTest() throws Exception {
         U.closeQuiet(conn);
 
         super.beforeTest();
+
+        Ignite ig = U.field(store, "ignite");
+
+        this.ig = ig;
+
+        binaryEnable = ig.configuration().getMarshaller() instanceof BinaryMarshaller;
     }
 
     /**
@@ -292,26 +309,48 @@ public void testLoadCache() throws Exception {
 
         U.closeQuiet(conn);
 
-        final Collection<OrganizationKey> orgKeys = new ConcurrentLinkedQueue<>();
-        final Collection<PersonKey> prnKeys = new ConcurrentLinkedQueue<>();
-        final Collection<PersonComplexKey> prnComplexKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> orgKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> prnKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> prnComplexKeys = new ConcurrentLinkedQueue<>();
 
         IgniteBiInClosure<Object, Object> c = new CI2<Object, Object>() {
             @Override public void apply(Object k, Object v) {
-                if (k instanceof OrganizationKey && v instanceof Organization)
-                    orgKeys.add((OrganizationKey)k);
-                else if (k instanceof PersonKey && v instanceof Person)
-                    prnKeys.add((PersonKey)k);
-                else if (k instanceof PersonComplexKey && v instanceof Person) {
-                    PersonComplexKey key = (PersonComplexKey)k;
-
-                    Person val = (Person)v;
-
-                    assertTrue("Key ID should be the same as value ID", key.getId() == val.getId());
-                    assertTrue("Key orgID should be the same as value orgID", key.getOrgId() == val.getOrgId());
-                    assertEquals("name" + key.getId(), val.getName());
-
-                    prnComplexKeys.add((PersonComplexKey)k);
+                if (binaryEnable){
+                    if (k instanceof BinaryObject && v instanceof BinaryObject) {
+                        BinaryObject key = (BinaryObject)k;
+                        BinaryObject val = (BinaryObject)v;
+
+                        String keyType = key.type().typeName();
+                        String valType = val.type().typeName();
+
+                        if (OrganizationKey.class.getName().equals(keyType)
+                            && Organization.class.getName().equals(valType))
+                            orgKeys.add(key);
+
+                        if (PersonKey.class.getName().equals(keyType)
+                            && Person.class.getName().equals(valType))
+                            prnKeys.add(key);
+
+                        if (PersonComplexKey.class.getName().equals(keyType)
+                            && Person.class.getName().equals(valType))
+                            prnComplexKeys.add(key);
+                    }
+                }else {
+                    if (k instanceof OrganizationKey && v instanceof Organization)
+                        orgKeys.add(k);
+                    else if (k instanceof PersonKey && v instanceof Person)
+                        prnKeys.add(k);
+                    else if (k instanceof PersonComplexKey && v instanceof Person) {
+                        PersonComplexKey key = (PersonComplexKey)k;
+
+                        Person val = (Person)v;
+
+                        assertTrue("Key ID should be the same as value ID", key.getId() == val.getId());
+                        assertTrue("Key orgID should be the same as value orgID", key.getOrgId() == val.getOrgId());
+                        assertEquals("name" + key.getId(), val.getName());
+
+                        prnComplexKeys.add(k);
+                    }
                 }
             }
         };
@@ -322,15 +361,16 @@ else if (k instanceof PersonComplexKey && v instanceof Person) {
         assertEquals(PERSON_CNT, prnKeys.size());
         assertEquals(PERSON_CNT, prnComplexKeys.size());
 
-        Collection<OrganizationKey> tmpOrgKeys = new ArrayList<>(orgKeys);
-        Collection<PersonKey> tmpPrnKeys = new ArrayList<>(prnKeys);
-        Collection<PersonComplexKey> tmpPrnComplexKeys = new ArrayList<>(prnComplexKeys);
+        Collection<Object> tmpOrgKeys = new ArrayList<>(orgKeys);
+        Collection<Object> tmpPrnKeys = new ArrayList<>(prnKeys);
+        Collection<Object> tmpPrnComplexKeys = new ArrayList<>(prnComplexKeys);
 
         orgKeys.clear();
         prnKeys.clear();
         prnComplexKeys.clear();
 
-        store.loadCache(c, OrganizationKey.class.getName(), "SELECT name, city, id FROM ORGANIZATION",
+        store.loadCache(
+            c, OrganizationKey.class.getName(), "SELECT name, city, id FROM ORGANIZATION",
             PersonKey.class.getName(), "SELECT org_id, id, name FROM Person WHERE id < 1000");
 
         assertEquals(ORGANIZATION_CNT, orgKeys.size());
@@ -380,14 +420,29 @@ public void testParallelLoad() throws Exception {
 
         U.closeQuiet(conn);
 
-        final Collection<PersonComplexKey> prnComplexKeys = new ConcurrentLinkedQueue<>();
+        final Collection<Object> prnComplexKeys = new ConcurrentLinkedQueue<>();
 
         IgniteBiInClosure<Object, Object> c = new CI2<Object, Object>() {
             @Override public void apply(Object k, Object v) {
-                if (k instanceof PersonComplexKey && v instanceof Person)
-                    prnComplexKeys.add((PersonComplexKey)k);
-                else
-                    fail("Unexpected entry [key=" + k + ", value=" + v + "]");
+                if (binaryEnable) {
+                    if (k instanceof BinaryObject && v instanceof BinaryObject) {
+                        BinaryObject key = (BinaryObject)k;
+                        BinaryObject val = (BinaryObject)v;
+
+                        String keyType = key.type().typeName();
+                        String valType = val.type().typeName();
+
+                        if (PersonComplexKey.class.getName().equals(keyType)
+                            && Person.class.getName().equals(valType))
+                            prnComplexKeys.add(key);
+                    }
+                }
+                else {
+                    if (k instanceof PersonComplexKey && v instanceof Person)
+                        prnComplexKeys.add(k);
+                    else
+                        fail("Unexpected entry [key=" + k + ", value=" + v + "]");
+                }
             }
         };
 
@@ -440,7 +495,7 @@ public void testWriteRetry() throws Exception {
         ses.newSession(null);
 
         try {
-            store.write(new CacheEntryImpl<>(k1, v1));
+            store.write(new CacheEntryImpl<>(wrap(k1), wrap(v1)));
 
             fail("CacheWriterException wasn't thrown.");
         }
@@ -469,4 +524,29 @@ public void testTimestamp() throws Exception {
 
         assertNull(store.load(k));
     }
+
+    /**
+     * @param obj Object.
+     */
+    private Object wrap(Object obj) throws IllegalAccessException {
+        if (binaryEnable) {
+            Class<?> cls = obj.getClass();
+
+            BinaryObjectBuilder builder = ig.binary().builder(cls.getName());
+
+            for (Field f : cls.getDeclaredFields()) {
+                if (f.getName().contains("serialVersionUID"))
+                    continue;
+
+                f.setAccessible(true);
+
+                builder.setField(f.getName(), f.get(obj));
+            }
+
+            return builder.build();
+        }
+
+        return obj;
+    }
+
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
index 5722fa3d6056a..14ba030e24590 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.binary.BinaryCachingMetadataHandler;
 import org.apache.ignite.internal.binary.BinaryContext;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl;
 import org.apache.ignite.internal.processors.cacheobject.NoOpBinary;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -327,6 +328,9 @@ public IgniteMock(
                     return typeName.hashCode();
                 }
             };
+
+            if (marshaller instanceof BinaryMarshaller)
+                ctx.configure((BinaryMarshaller)marshaller, configuration());
         }
 
         binaryMock = new NoOpBinary() {
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index deec72af95cd8..dce93747f12bd 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -32,6 +32,8 @@
 import org.apache.ignite.cache.store.GridCacheLoadOnlyStoreAdapterSelfTest;
 import org.apache.ignite.cache.store.StoreResourceInjectionSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerSelfTest;
+import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest;
+import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreMultitreadedSelfTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreOptimizedMarshallerSelfTest;
@@ -252,7 +254,9 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(CacheJdbcPojoStoreOptimizedMarshallerSelfTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreOptimizedMarshallerWithSqlEscapeSelfTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerSelfTest.class);
+        suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest.class);
+        suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.class);
         suite.addTestSuite(CacheJdbcPojoStoreMultitreadedSelfTest.class);
         suite.addTestSuite(GridCacheBalancingStoreSelfTest.class);
         suite.addTestSuite(GridCacheAffinityApiSelfTest.class);

From 8aa662dbed2f9ce27931920ace6efafbf0fc6671 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 13 Apr 2017 16:36:54 +0300
Subject: [PATCH 382/516] GG-12773: Fixed potential deadlock when service is
 being deployed while node is stopping.

(cherry picked from commit 9ad68c7)
---
 .../service/GridServiceProcessor.java         | 171 ++++++++++--------
 .../GridServiceProcessorStopSelfTest.java     | 137 +++++++++++++-
 2 files changed, 228 insertions(+), 80 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index c961eb0443cdc..7b6311c4018a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -493,124 +493,137 @@ public IgniteInternalFuture<?> deployKeyAffinitySingleton(String name, Service s
     public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
         A.notNull(cfg, "cfg");
 
-        ServicesCompatibilityState state = markCompatibilityStateAsUsed();
+        if (!busyLock.enterBusy()) {
+            IgniteCheckedException e = new IgniteCheckedException("Service deployment has been cancelled (node is stopping): " +
+                cfg.getName());
 
-        validate(cfg);
-
-        ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
+            return new GridFinishedFuture<>(e);
+        }
 
-        if (!state.srvcCompatibility) {
-            Marshaller marsh = ctx.config().getMarshaller();
+        try {
 
-            LazyServiceConfiguration cfg0;
+            ServicesCompatibilityState state = markCompatibilityStateAsUsed();
 
-            try {
-                byte[] srvcBytes = U.marshal(marsh, cfg.getService());
+            validate(cfg);
 
-                cfg0 = new LazyServiceConfiguration(cfg, srvcBytes);
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to marshal service with configured marshaller [srvc=" + cfg.getService()
-                    + ", marsh=" + marsh + "]", e);
+            ctx.security().authorize(cfg.getName(), SecurityPermission.SERVICE_DEPLOY, null);
 
-                return new GridFinishedFuture<>(e);
-            }
+            if (!state.srvcCompatibility) {
+                Marshaller marsh = ctx.config().getMarshaller();
 
-            cfg = cfg0;
-        }
+                LazyServiceConfiguration cfg0;
 
-        GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
+                try {
+                    byte[] srvcBytes = U.marshal(marsh, cfg.getService());
 
-        GridServiceDeploymentFuture old = depFuts.putIfAbsent(cfg.getName(), fut);
+                    cfg0 = new LazyServiceConfiguration(cfg, srvcBytes);
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to marshal service with configured marshaller [srvc=" + cfg.getService()
+                        + ", marsh=" + marsh + "]", e);
 
-        if (old != null) {
-            if (!old.configuration().equalsIgnoreNodeFilter(cfg)) {
-                fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
-                    "different configuration) [deployed=" + old.configuration() + ", new=" + cfg + ']'));
+                    return new GridFinishedFuture<>(e);
+                }
 
-                return fut;
+                cfg = cfg0;
             }
 
-            return old;
-        }
+            GridServiceDeploymentFuture fut = new GridServiceDeploymentFuture(cfg);
 
-        if (ctx.clientDisconnected()) {
-            fut.onDone(new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
-                "Failed to deploy service, client node disconnected."));
+            GridServiceDeploymentFuture old = depFuts.putIfAbsent(cfg.getName(), fut);
 
-            depFuts.remove(cfg.getName(), fut);
-        }
+            if (old != null) {
+                if (!old.configuration().equalsIgnoreNodeFilter(cfg)) {
+                    fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
+                        "different configuration) [deployed=" + old.configuration() + ", new=" + cfg + ']'));
 
-        while (true) {
-            try {
-                GridServiceDeploymentKey key = new GridServiceDeploymentKey(cfg.getName());
+                    return fut;
+                }
+
+                return old;
+            }
 
-                if (ctx.deploy().enabled())
-                    ctx.cache().context().deploy().ignoreOwnership(true);
+            if (ctx.clientDisconnected()) {
+                fut.onDone(new IgniteClientDisconnectedCheckedException(ctx.cluster().clientReconnectFuture(),
+                    "Failed to deploy service, client node disconnected."));
 
+                depFuts.remove(cfg.getName(), fut);
+            }
+
+            while (true) {
                 try {
-                    GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
-                        new GridServiceDeployment(ctx.localNodeId(), cfg));
+                    GridServiceDeploymentKey key = new GridServiceDeploymentKey(cfg.getName());
 
-                    if (dep != null) {
-                        if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
-                            // Remove future from local map.
-                            depFuts.remove(cfg.getName(), fut);
+                    if (ctx.deploy().enabled())
+                        ctx.cache().context().deploy().ignoreOwnership(true);
 
-                            fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
-                                "different configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']'));
-                        }
-                        else {
-                            Iterator<Cache.Entry<Object, Object>> it = serviceEntries(
-                                ServiceAssignmentsPredicate.INSTANCE);
+                    try {
+                        GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
+                            new GridServiceDeployment(ctx.localNodeId(), cfg));
+
+                        if (dep != null) {
+                            if (!dep.configuration().equalsIgnoreNodeFilter(cfg)) {
+                                // Remove future from local map.
+                                depFuts.remove(cfg.getName(), fut);
+
+                                fut.onDone(new IgniteCheckedException("Failed to deploy service (service already exists with " +
+                                    "different configuration) [deployed=" + dep.configuration() + ", new=" + cfg + ']'));
+                            }
+                            else {
+                                Iterator<Cache.Entry<Object, Object>> it = serviceEntries(
+                                    ServiceAssignmentsPredicate.INSTANCE);
 
-                            while (it.hasNext()) {
-                                Cache.Entry<Object, Object> e = it.next();
+                                while (it.hasNext()) {
+                                    Cache.Entry<Object, Object> e = it.next();
 
-                                if (e.getKey() instanceof GridServiceAssignmentsKey) {
-                                    GridServiceAssignments assigns = (GridServiceAssignments)e.getValue();
+                                    if (e.getKey() instanceof GridServiceAssignmentsKey) {
+                                        GridServiceAssignments assigns = (GridServiceAssignments)e.getValue();
 
-                                    if (assigns.name().equals(cfg.getName())) {
-                                        // Remove future from local map.
-                                        depFuts.remove(cfg.getName(), fut);
+                                        if (assigns.name().equals(cfg.getName())) {
+                                            // Remove future from local map.
+                                            depFuts.remove(cfg.getName(), fut);
 
-                                        fut.onDone();
+                                            fut.onDone();
 
-                                        break;
+                                            break;
+                                        }
                                     }
                                 }
-                            }
 
-                            if (!dep.configuration().equalsIgnoreNodeFilter(cfg))
-                                U.warn(log, "Service already deployed with different configuration (will ignore) " +
-                                    "[deployed=" + dep.configuration() + ", new=" + cfg + ']');
+                                if (!dep.configuration().equalsIgnoreNodeFilter(cfg))
+                                    U.warn(log, "Service already deployed with different configuration (will ignore) " +
+                                        "[deployed=" + dep.configuration() + ", new=" + cfg + ']');
+                            }
                         }
                     }
-                }
-                finally {
-                    if (ctx.deploy().enabled())
-                        ctx.cache().context().deploy().ignoreOwnership(false);
-                }
+                    finally {
+                        if (ctx.deploy().enabled())
+                            ctx.cache().context().deploy().ignoreOwnership(false);
+                    }
 
-                return fut;
-            }
-            catch (ClusterTopologyCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
-            }
-            catch (IgniteCheckedException e) {
-                if (e.hasCause(ClusterTopologyCheckedException.class)) {
+                    return fut;
+                }
+                catch (ClusterTopologyCheckedException e) {
                     if (log.isDebugEnabled())
                         log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
-
-                    continue;
                 }
+                catch (IgniteCheckedException e) {
+                    if (e.hasCause(ClusterTopologyCheckedException.class)) {
+                        if (log.isDebugEnabled())
+                            log.debug("Topology changed while deploying service (will retry): " + e.getMessage());
 
-                U.error(log, "Failed to deploy service: " + cfg.getName(), e);
+                        continue;
+                    }
 
-                return new GridFinishedFuture<>(e);
+                    U.error(log, "Failed to deploy service: " + cfg.getName(), e);
+
+                    return new GridFinishedFuture<>(e);
+                }
             }
         }
+        finally {
+            busyLock.leaveBusy();
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
index 03b00f47e6c0d..dc2dd1262c775 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorStopSelfTest.java
@@ -21,6 +21,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
@@ -31,8 +32,12 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.lang.GridPlainCallable;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
 import org.apache.ignite.services.ServiceContext;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -161,8 +166,138 @@ public void testStopDuringHangedDeployment() throws Exception {
         assertTrue("Deploy future isn't completed", wait);
 
         fut.get();
+    }
 
-        Ignition.stopAll(true);
+    /**
+     * @throws Exception If fails
+     */
+    public void testServiceDeploymentCancelationOnStop() throws Exception {
+        final Ignite node = startGrid(0);
+
+        final IgniteServices services = node.services();
+        // Deploy some service.
+        services.deploy(getServiceConfiguration("myService1"));
+
+        //Stop node async, this will cancel the service #1.
+        final IgniteInternalFuture<Boolean> stopAsync = GridTestUtils.runAsync(new GridPlainCallable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                node.close();
+
+                return true;
+            }
+        }, "node-stopping-thread");
+
+        // Wait for the service #1 cancellation during node stopping.
+        // At this point node.stop process will be paused until svcCancelFinishLatch released.
+        ServiceImpl.svcCancelStartLatch.await();
+
+        final AtomicReference<IgniteFuture> queuedFuture = new AtomicReference<>();
+
+        // Try to deploy another service.
+        final IgniteInternalFuture<?> deployAsync = GridTestUtils.runAsync(new GridPlainCallable<Boolean>() {
+            @Override public Boolean call() throws Exception {
+                IgniteServices async = services.withAsync();
+
+                async.deploy(getServiceConfiguration("myService2"));
+
+                IgniteFuture<Object> future = async.future();
+
+                queuedFuture.set(future);
+
+                // Here, deployment future is added to queue and
+                // then it will be cancelled when svcCancelFinishLatch be released.
+                // So, we'll wait for queue cleaning and try to deploy one more service.
+                try {
+                    future.get();
+                }
+                catch (Exception ignore) {
+                    // Noop.
+                }
+
+                // Normally, this should fail with some Exception as node is stopping right now.
+                // But we face a deadlock here.
+                for (int i = 0; i < 5; i++) {
+                    try {
+                        services.deploy(getServiceConfiguration("service3"));
+                    }
+                    catch (Exception ignore) {
+                        // Noop.
+                    }
+                }
+
+                return true;
+            }
+        }, "svc-deploy-thread");
+
+        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return queuedFuture.get() != null;
+            }
+        }, 3000);
+
+        // Allow node to be stopped.
+        ServiceImpl.svcCancelFinishLatch.countDown();
+
+        // Wait for all service deployments have finished.
+        boolean deployDone = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                IgniteFuture fut = queuedFuture.get();
+
+                return fut != null && fut.isDone() && deployAsync.isDone();
+
+            }
+        }, 5000);
+
+        assertTrue("Node stopping and service deployment processes falls into a deadlock.", deployDone);
+
+        if (!deployDone)
+            deployAsync.cancel();
+
+        if (!stopAsync.isDone())
+            stopAsync.cancel();
+    }
+
+    /** */
+    private ServiceConfiguration getServiceConfiguration(String svcName) {
+        ServiceConfiguration svcCfg = new ServiceConfiguration();
+        svcCfg.setName(svcName);
+        svcCfg.setService(new ServiceImpl());
+        svcCfg.setTotalCount(1);
+
+        return svcCfg;
+    }
+
+    /** Dummy Implementation. */
+    static class ServiceImpl implements Service {
+        /** */
+        static final CountDownLatch svcCancelStartLatch = new CountDownLatch(1);
+
+        /** */
+        static final CountDownLatch svcCancelFinishLatch = new CountDownLatch(1);
+
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+            System.out.println("cancel service: " + ctx.executionId());
+            try {
+                svcCancelStartLatch.countDown();
+
+                svcCancelFinishLatch.await();
+            }
+            catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(ServiceContext ctx) throws Exception {
+            System.out.println("init service: " + ctx.executionId());
+            // No-op
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+            // No-op
+        }
     }
 
     /**

From 87ddd3f87ba6746307a7ba54947ffd1d4b5c6076 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 22 Sep 2017 16:50:27 +0300
Subject: [PATCH 383/516] GG-12755 - Backport ExchangeActions from master

---
 .../cache/CacheAffinitySharedManager.java     | 215 +++++++++-------
 .../cache/DynamicCacheChangeBatch.java        |  26 +-
 .../processors/cache/ExchangeActions.java     | 240 ++++++++++++++++++
 .../GridCachePartitionExchangeManager.java    |  20 +-
 .../processors/cache/GridCacheProcessor.java  |  94 ++++---
 .../dht/GridDhtAssignmentFetchFuture.java     |  11 +-
 .../GridDhtPartitionsExchangeFuture.java      | 156 +++++-------
 7 files changed, 522 insertions(+), 240 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 7cbd333907058..de2390ece1703 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -381,36 +381,17 @@ public void forceCloseCache(final GridDhtPartitionsExchangeFuture fut, boolean c
      *
      * @param fut Exchange future.
      * @param crd Coordinator flag.
-     * @param reqs Cache change requests.
+     * @param exchActions Exchange actions.
      * @throws IgniteCheckedException If failed.
      * @return {@code True} if client-only exchange is needed.
      */
     public boolean onCacheChangeRequest(final GridDhtPartitionsExchangeFuture fut,
         boolean crd,
-        Collection<DynamicCacheChangeRequest> reqs)
+        final ExchangeActions exchActions)
         throws IgniteCheckedException {
-        assert !F.isEmpty(reqs) : fut;
-
-        for (DynamicCacheChangeRequest req : reqs) {
-            Integer cacheId = CU.cacheId(req.cacheName());
-
-            if (req.stop()) {
-                DynamicCacheDescriptor desc = registeredCaches.remove(cacheId);
-
-                assert desc != null : cacheId;
-            }
-            else if (req.start() && !req.clientStartOnly()) {
-                DynamicCacheDescriptor desc = new DynamicCacheDescriptor(cctx.kernalContext(),
-                    req.startCacheConfiguration(),
-                    req.cacheType(),
-                    false,
-                    req.deploymentId());
+        assert exchActions != null && !exchActions.empty() : exchActions;
 
-                DynamicCacheDescriptor old = registeredCaches.put(cacheId, desc);
-
-                assert old == null : old;
-            }
-        }
+        updateCachesInfo(exchActions);
 
         boolean clientOnly = true;
 
@@ -426,82 +407,115 @@ else if (req.start() && !req.clientStartOnly()) {
 
         Set<Integer> stoppedCaches = null;
 
-        for (DynamicCacheChangeRequest req : reqs) {
-            if (!(req.clientStartOnly() || req.close()))
+        for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) {
+            DynamicCacheDescriptor cacheDesc = action.descriptor();
+
+            DynamicCacheChangeRequest req = action.request();
+
+            Integer cacheId = cacheDesc.cacheId();
+
+            cctx.cache().prepareCacheStart(cacheDesc, req, fut.topologyVersion());
+
+            if (!req.clientStartOnly())
                 clientOnly = false;
 
-            Integer cacheId = CU.cacheId(req.cacheName());
+            if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
+                if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
+                    U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+            }
+
+            if (!crd || !lateAffAssign) {
+                GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+
+                if (cacheCtx != null && !cacheCtx.isLocal()) {
+                    boolean clientCacheStarted =
+                        req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
 
-            if (req.start()) {
-                cctx.cache().prepareCacheStart(req, fut.topologyVersion());
+                    if (clientCacheStarted)
+                        initAffinity(cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
+                    else if (!req.clientStartOnly()) {
+                        assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
 
-                if (fut.isCacheAdded(cacheId, fut.topologyVersion())) {
-                    if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty())
-                        U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
+                        GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
+
+                        assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
+
+                        List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
+                            fut.discoveryEvent(), fut.discoCache());
+
+                        aff.initialize(fut.topologyVersion(), assignment);
+                    }
                 }
+            }
+            else
+                initStartedCacheOnCoordinator(fut, action.descriptor());
+        }
 
-                if (!crd || !lateAffAssign) {
-                    GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+        for (DynamicCacheChangeRequest req : exchActions.cacheCloseRequests(cctx.localNodeId())) {
+            Integer cacheId = CU.cacheId(req.cacheName());
 
-                    if (cacheCtx != null && !cacheCtx.isLocal()) {
-                        boolean clientCacheStarted =
-                            req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId());
+            cctx.cache().blockGateway(req);
 
-                        if (clientCacheStarted)
-                            initAffinity(cacheCtx.affinity().affinityCache(), fut, lateAffAssign);
-                        else if (!req.clientStartOnly()) {
-                            assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion());
+            if (crd) {
+                GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
-                            GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache();
+                // Client cache was stopped, need create 'client' CacheHolder.
+                if (cacheCtx != null && !cacheCtx.affinityNode()) {
+                    CacheHolder cache = caches.remove(cacheId);
 
-                            assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion();
+                    assert !cache.client() : cache;
 
-                            List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(),
-                                fut.discoveryEvent(), fut.discoCache());
+                    cache = CacheHolder2.create(cctx,
+                        cctx.cache().cacheDescriptor(cacheId),
+                        fut,
+                        cache.affinity());
 
-                            aff.initialize(fut.topologyVersion(), assignment);
-                        }
-                    }
+                    caches.put(cacheId, cache);
                 }
-                else
-                    initStartedCacheOnCoordinator(fut, cacheId);
             }
-            else if (req.stop() || req.close()) {
-                cctx.cache().blockGateway(req);
+        }
 
-                if (crd) {
-                    boolean rmvCache = false;
+        for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) {
+            DynamicCacheChangeRequest req = action.request();
 
-                    if (req.close() && req.initiatingNodeId().equals(cctx.localNodeId())) {
-                        GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+            clientOnly = false;
 
-                        rmvCache = cacheCtx != null && !cacheCtx.affinityNode();
-                    }
-                    else if (req.stop())
-                        rmvCache = true;
+            Integer cacheId = CU.cacheId(req.cacheName());
 
-                    if (rmvCache) {
-                        CacheHolder cache = caches.remove(cacheId);
+            cctx.cache().blockGateway(req);
 
-                        if (cache != null) {
-                            if (!req.stop()) {
-                                assert !cache.client();
+            if (crd) {
+                boolean rmvCache = false;
 
-                                cache = CacheHolder2.create(cctx,
-                                    cctx.cache().cacheDescriptor(cacheId),
-                                    fut,
-                                    cache.affinity());
+                if (req.close() && req.initiatingNodeId().equals(cctx.localNodeId())) {
+                    GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
-                                caches.put(cacheId, cache);
-                            }
-                            else {
-                                if (stoppedCaches == null)
-                                    stoppedCaches = new HashSet<>();
+                    rmvCache = cacheCtx != null && !cacheCtx.affinityNode();
+                }
+                else if (req.stop())
+                    rmvCache = true;
 
-                                stoppedCaches.add(cache.cacheId());
+                if (rmvCache) {
+                    CacheHolder cache = caches.remove(cacheId);
 
-                                cctx.io().removeHandler(cacheId, GridDhtAffinityAssignmentResponse.class);
-                            }
+                    if (cache != null) {
+                        if (!req.stop()) {
+                            assert !cache.client();
+
+                            cache = CacheHolder2.create(cctx,
+                                cctx.cache().cacheDescriptor(cacheId),
+                                fut,
+                                cache.affinity());
+
+                            caches.put(cacheId, cache);
+                        }
+                        else {
+                            if (stoppedCaches == null)
+                                stoppedCaches = new HashSet<>();
+
+                            stoppedCaches.add(cache.cacheId());
+
+                            cctx.io().removeHandler(cacheId, GridDhtAffinityAssignmentResponse.class);
                         }
                     }
                 }
@@ -539,6 +553,37 @@ else if (req.stop())
         return clientOnly;
     }
 
+    /**
+     * @param exchActions Cache change requests to execute on exchange.
+     */
+    private void updateCachesInfo(ExchangeActions exchActions) {
+        for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) {
+            Integer cacheId = CU.cacheId(action.request().cacheName());
+
+            DynamicCacheDescriptor desc = registeredCaches.remove(cacheId);
+
+            assert desc != null : cacheId;
+        }
+
+        for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) {
+            DynamicCacheChangeRequest req = action.request();
+
+            Integer cacheId = CU.cacheId(req.cacheName());
+
+            if (!req.clientStartOnly()) {
+                DynamicCacheDescriptor desc = new DynamicCacheDescriptor(cctx.kernalContext(),
+                    req.startCacheConfiguration(),
+                    req.cacheType(),
+                    false,
+                    req.deploymentId());
+
+                DynamicCacheDescriptor old = registeredCaches.put(cacheId, desc);
+
+                assert old == null : old;
+            }
+        }
+    }
+
     /**
      * Called when received {@link CacheAffinityChangeMessage} which should complete exchange.
      *
@@ -784,20 +829,20 @@ private void forAllCaches(boolean crd, IgniteInClosureX<GridAffinityAssignmentCa
 
     /**
      * @param fut Exchange future.
-     * @param cacheId Cache ID.
+     * @param desc Dynamic cache descriptor.
      * @throws IgniteCheckedException If failed.
      */
-    private void initStartedCacheOnCoordinator(GridDhtPartitionsExchangeFuture fut, final Integer cacheId)
+    private void initStartedCacheOnCoordinator(GridDhtPartitionsExchangeFuture fut, final DynamicCacheDescriptor desc)
         throws IgniteCheckedException {
+        assert desc != null;
+
+        Integer cacheId = desc.cacheId();
+
         CacheHolder cache = caches.get(cacheId);
 
         GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
         if (cache == null) {
-            DynamicCacheDescriptor desc = cctx.cache().cacheDescriptor(cacheId);
-
-            assert desc != null : cacheId;
-
             if (desc.cacheConfiguration().getCacheMode() == LOCAL)
                 return;
 
@@ -877,7 +922,7 @@ private void initAffinity(GridAffinityAssignmentCache aff, GridDhtPartitionsExch
         }
         else {
             GridDhtAssignmentFetchFuture fetchFut = new GridDhtAssignmentFetchFuture(cctx,
-                aff.cacheName(),
+                aff.cacheId(),
                 fut.topologyVersion(),
                 fut.discoCache());
 
@@ -1006,7 +1051,7 @@ private void fetchAffinityOnJoin(GridDhtPartitionsExchangeFuture fut) throws Ign
             }
             else {
                 GridDhtAssignmentFetchFuture fetchFut = new GridDhtAssignmentFetchFuture(cctx,
-                    cacheCtx.name(),
+                    cacheCtx.cacheId(),
                     topVer,
                     fut.discoCache());
 
@@ -1173,7 +1218,7 @@ private IgniteInternalFuture<?> initCoordinatorCaches(final GridDhtPartitionsExc
                     assert prev.topologyVersion().compareTo(fut.topologyVersion()) < 0 : prev;
 
                     GridDhtAssignmentFetchFuture fetchFut = new GridDhtAssignmentFetchFuture(cctx,
-                        aff.cacheName(),
+                        aff.cacheId(),
                         prev.topologyVersion(),
                         prev.discoCache());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
index a2500633a1ccf..d9d8b0ca9e4df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
@@ -47,6 +47,9 @@ public class DynamicCacheChangeBatch implements DiscoveryCustomMessage {
     /** */
     private boolean clientReconnect;
 
+    /** */
+    private transient ExchangeActions exchangeActions;
+
     /**
      * @param reqs Requests.
      */
@@ -117,14 +120,23 @@ public boolean clientReconnect() {
      * @return {@code True} if request should trigger partition exchange.
      */
     public boolean exchangeNeeded() {
-        if (reqs != null) {
-            for (DynamicCacheChangeRequest req : reqs) {
-                if (req.exchangeNeeded())
-                    return true;
-            }
-        }
+        return exchangeActions != null;
+    }
 
-        return false;
+    /**
+     * @return Exchange actions.
+     */
+    public ExchangeActions exchangeActions() {
+        return exchangeActions;
+    }
+
+    /**
+     * @param exchangeActions New exchange actions.
+     */
+    public void exchangeActions(ExchangeActions exchangeActions) {
+        assert exchangeActions != null && !exchangeActions.empty() : exchangeActions;
+
+        this.exchangeActions = exchangeActions;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
new file mode 100644
index 0000000000000..0f5f914a2c003
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.F;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Cache change requests to execute when receive {@link DynamicCacheChangeBatch} event.
+ */
+public class ExchangeActions {
+    /** */
+    private Map<String, CacheActionData> cachesToStart;
+
+    /** */
+    private Map<String, CacheActionData> cachesToStop;
+
+    /** */
+    private Map<String, CacheActionData> cachesToClose;
+
+    /**
+     * @return New caches start requests.
+     */
+    public Collection<CacheActionData> cacheStartRequests() {
+        return cachesToStart != null ? cachesToStart.values() : Collections.<CacheActionData>emptyList();
+    }
+
+    /**
+     * @return Stop cache requests.
+     */
+    public Collection<CacheActionData> cacheStopRequests() {
+        return cachesToStop != null ? cachesToStop.values() : Collections.<CacheActionData>emptyList();
+    }
+
+    /**
+     * @return {@code True} if have start requests.
+     */
+    public boolean hasStart() {
+        return !F.isEmpty(cachesToStart);
+    }
+
+    /**
+     * @param cacheId Cache ID.
+     * @return {@code True} if cache stop was requested.
+     */
+    public boolean cacheStopped(int cacheId) {
+        if (cachesToStop != null) {
+            for (CacheActionData cache : cachesToStop.values()) {
+                if (cache.desc.cacheId() == cacheId)
+                    return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * @param cacheId Cache ID.
+     * @return {@code True} if cache start was requested.
+     */
+    public boolean cacheStarted(int cacheId) {
+        if (cachesToStart != null) {
+            for (CacheActionData cache : cachesToStart.values()) {
+                if (cache.desc.cacheId() == cacheId)
+                    return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * @param map Actions map.
+     * @param req Request.
+     * @param desc Cache descriptor.
+     * @return Actions map.
+     */
+    private Map<String, CacheActionData> add(Map<String, CacheActionData> map,
+        DynamicCacheChangeRequest req,
+        DynamicCacheDescriptor desc) {
+        assert req != null;
+        assert desc != null;
+
+        if (map == null)
+            map = new LinkedHashMap<>();
+
+        CacheActionData old = map.put(req.cacheName(), new CacheActionData(req, desc));
+
+        assert old == null : old;
+
+        return map;
+    }
+
+    /**
+     * @param req Request.
+     * @param desc Cache descriptor.
+     */
+    void addCacheToStart(DynamicCacheChangeRequest req, DynamicCacheDescriptor desc) {
+        assert req.start() : req;
+
+        cachesToStart = add(cachesToStart, req, desc);
+    }
+
+    /**
+     * @param req Request.
+     * @param desc Cache descriptor.
+     */
+    void addCacheToStop(DynamicCacheChangeRequest req, DynamicCacheDescriptor desc) {
+        assert req.stop() : req;
+
+        cachesToStop = add(cachesToStop, req, desc);
+    }
+
+    /**
+     * @param req Request.
+     * @param desc Cache descriptor.
+     */
+    void addCacheToClose(DynamicCacheChangeRequest req, DynamicCacheDescriptor desc) {
+        assert req.close() : req;
+
+        cachesToClose = add(cachesToClose, req, desc);
+    }
+
+    /**
+     * @param nodeId Local node ID.
+     * @return Close cache requests.
+     */
+    List<DynamicCacheChangeRequest> cacheCloseRequests(UUID nodeId) {
+        List<DynamicCacheChangeRequest> res = null;
+
+        if (cachesToClose != null) {
+            for (CacheActionData req : cachesToClose.values()) {
+                if (nodeId.equals(req.req.initiatingNodeId())) {
+                    if (res == null)
+                        res = new ArrayList<>(cachesToClose.size());
+
+                    res.add(req.req);
+                }
+            }
+        }
+
+        return res != null ? res : Collections.<DynamicCacheChangeRequest>emptyList();
+    }
+
+    /**
+     * @return {@code True} if there are no cache change actions.
+     */
+    public boolean empty() {
+        return F.isEmpty(cachesToStart) &&
+            F.isEmpty(cachesToStop) &&
+            F.isEmpty(cachesToClose);
+    }
+
+    /**
+     * @param ctx Context.
+     */
+    public void completeRequestFutures(GridCacheSharedContext ctx, @Nullable Throwable err) {
+        completeRequestFutures(cachesToStart, ctx, err);
+        completeRequestFutures(cachesToStop, ctx, err);
+        completeRequestFutures(cachesToClose, ctx, err);
+    }
+
+    /**
+     * @param map Actions map.
+     * @param ctx Context.
+     */
+    private void completeRequestFutures(Map<String, CacheActionData> map, GridCacheSharedContext ctx, @Nullable Throwable err) {
+        if (map != null) {
+            for (CacheActionData req : map.values())
+                ctx.cache().completeStartFuture(req.req, err);
+        }
+    }
+
+    /**
+     *
+     */
+    public static class CacheActionData {
+        /** */
+        private final DynamicCacheChangeRequest req;
+
+        /** */
+        private final DynamicCacheDescriptor desc;
+
+        /**
+         * @param req Request.
+         * @param desc Cache descriptor.
+         */
+        CacheActionData(DynamicCacheChangeRequest req, DynamicCacheDescriptor desc) {
+            assert req != null;
+            assert desc != null;
+
+            this.req = req;
+            this.desc = desc;
+        }
+
+        /**
+         * @return Request.
+         */
+        public DynamicCacheChangeRequest request() {
+            return req;
+        }
+
+        /**
+         * @return Cache descriptor.
+         */
+        public DynamicCacheDescriptor descriptor() {
+            return desc;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "ExchangeActions [startCaches=" + (cachesToStart != null ? cachesToStart.keySet() : null) +
+            ", stopCaches=" + (cachesToStop != null ? cachesToStop.keySet() : null) +
+            ", closeCaches=" + (cachesToClose != null ? cachesToClose.keySet() : null)
+            + ']';
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index d146099243916..a41004bc771c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -229,13 +229,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                     if (customEvt.customMessage() instanceof DynamicCacheChangeBatch) {
                         DynamicCacheChangeBatch batch = (DynamicCacheChangeBatch)customEvt.customMessage();
 
-                        Collection<DynamicCacheChangeRequest> valid = new ArrayList<>(batch.requests().size());
-
                         // Validate requests to check if event should trigger partition exchange.
                         for (final DynamicCacheChangeRequest req : batch.requests()) {
-                            if (req.exchangeNeeded())
-                                valid.add(req);
-                            else {
+                            if (!req.exchangeNeeded()) {
                                 IgniteInternalFuture<?> fut = null;
 
                                 if (req.cacheFutureTopologyVersion() != null)
@@ -253,10 +249,10 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                             }
                         }
 
-                        if (!F.isEmpty(valid)) {
+                        if (batch.exchangeNeeded()) {
                             exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type());
 
-                            exchFut = exchangeFuture(exchId, evt, cache, valid, null);
+                            exchFut = exchangeFuture(exchId, evt, cache, batch.exchangeActions(), null);
                         }
                     }
                     else if (customEvt.customMessage() instanceof CacheAffinityChangeMessage) {
@@ -1091,25 +1087,25 @@ private GridDhtPartitionExchangeId exchangeId(UUID nodeId, AffinityTopologyVersi
      * @param exchId Exchange ID.
      * @param discoEvt Discovery event.
      * @param cache Discovery data cache.
-     * @param reqs Cache change requests.
+     * @param exchActions Exchange actions.
      * @param affChangeMsg Affinity change message.
      * @return Exchange future.
      */
     private GridDhtPartitionsExchangeFuture exchangeFuture(GridDhtPartitionExchangeId exchId,
         @Nullable DiscoveryEvent discoEvt,
         @Nullable DiscoCache cache,
-        @Nullable Collection<DynamicCacheChangeRequest> reqs,
+        @Nullable ExchangeActions exchActions,
         @Nullable CacheAffinityChangeMessage affChangeMsg) {
         GridDhtPartitionsExchangeFuture fut;
 
         GridDhtPartitionsExchangeFuture old = exchFuts.addx(
-            fut = new GridDhtPartitionsExchangeFuture(cctx, busyLock, exchId, reqs, affChangeMsg));
+            fut = new GridDhtPartitionsExchangeFuture(cctx, busyLock, exchId, exchActions, affChangeMsg));
 
         if (old != null) {
             fut = old;
 
-            if (reqs != null)
-                fut.cacheChangeRequests(reqs);
+            if (exchActions != null)
+                fut.exchangeActions(exchActions);
 
             if (affChangeMsg != null)
                 fut.affinityChangeMessage(affChangeMsg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 96473e1f043d2..488cc2dd37e23 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -39,7 +39,6 @@
 import javax.cache.configuration.Factory;
 import javax.cache.integration.CacheLoader;
 import javax.cache.integration.CacheWriter;
-import javax.management.JMException;
 import javax.management.MBeanServer;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -63,7 +62,6 @@
 import org.apache.ignite.configuration.TransactionConfiguration;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteComponentType;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -102,6 +100,7 @@
 import org.apache.ignite.internal.processors.plugin.CachePluginManager;
 import org.apache.ignite.internal.processors.query.GridQueryProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
+import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
 import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -1628,13 +1627,13 @@ public CacheMode cacheMode(String cacheName) {
      * @param topVer Topology version.
      * @throws IgniteCheckedException If failed.
      */
-    public void prepareCacheStart(DynamicCacheChangeRequest req, AffinityTopologyVersion topVer)
+    public void prepareCacheStart(DynamicCacheDescriptor cacheDesc,
+        DynamicCacheChangeRequest req,
+        AffinityTopologyVersion topVer)
         throws IgniteCheckedException {
         assert req.start() : req;
         assert req.cacheType() != null : req;
 
-        String cacheName = maskNull(req.cacheName());
-
         prepareCacheStart(
             req.startCacheConfiguration(),
             req.nearCacheConfiguration(),
@@ -1645,10 +1644,8 @@ public void prepareCacheStart(DynamicCacheChangeRequest req, AffinityTopologyVer
             topVer
         );
 
-        DynamicCacheDescriptor desc = registeredCaches.get(cacheName);
-
-        if (desc != null)
-            desc.onStart();
+        if (cacheDesc != null)
+            cacheDesc.onStart();
     }
 
     /**
@@ -1820,14 +1817,14 @@ private void prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose)
      * Callback invoked when first exchange future for dynamic cache is completed.
      *
      * @param topVer Completed topology version.
-     * @param reqs Change requests.
+     * @param exchActions Exchange actions.
      * @param err Error.
      */
     @SuppressWarnings("unchecked")
     public void onExchangeDone(
         AffinityTopologyVersion topVer,
-        Collection<DynamicCacheChangeRequest> reqs,
-        Throwable err
+        @Nullable ExchangeActions exchActions,
+        @Nullable Throwable err
     ) {
         for (GridCacheAdapter<?, ?> cache : caches.values()) {
             GridCacheContext<?, ?> cacheCtx = cache.context();
@@ -1842,36 +1839,36 @@ public void onExchangeDone(
             }
         }
 
-        if (!F.isEmpty(reqs)) {
-            for (DynamicCacheChangeRequest req : reqs) {
-                if (err == null) {
-                    String masked = maskNull(req.cacheName());
+        if (exchActions != null && err == null) {
+            for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) {
+                DynamicCacheChangeRequest req = action.request();
 
-                    if (req.stop()) {
-                        stopGateway(req);
+                stopGateway(req);
 
-                        prepareCacheStop(req, false);
-                    }
-                    else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId())) {
-                        IgniteCacheProxy<?, ?> proxy = jCacheProxies.remove(masked);
+                prepareCacheStop(req, false);
+            }
 
-                        if (proxy != null) {
-                            if (proxy.context().affinityNode()) {
-                                GridCacheAdapter<?, ?> cache = caches.get(masked);
+            for (DynamicCacheChangeRequest req : exchActions.cacheCloseRequests(ctx.localNodeId())) {
+                String cacheName = maskNull(req.cacheName());
 
-                                if (cache != null)
-                                    jCacheProxies.put(masked, new IgniteCacheProxy(cache.context(), cache, null, false));
-                            }
-                            else {
-                                proxy.context().gate().onStopped();
+                IgniteCacheProxy<?, ?> proxy = jCacheProxies.get(cacheName);
 
-                                prepareCacheStop(req, false);
-                            }
-                        }
+                if (proxy != null) {
+                    if (proxy.context().affinityNode()) {
+                        GridCacheAdapter<?, ?> cache = caches.get(cacheName);
+
+                        assert cache != null : cacheName;
+
+                        jCacheProxies.put(cacheName, new IgniteCacheProxy(cache.context(), cache, null, false));
                     }
-                }
+                    else {
+                        jCacheProxies.remove(cacheName);
 
-                completeStartFuture(req, err);
+                        proxy.context().gate().onStopped();
+
+                        prepareCacheStop(req, false);
+                    }
+                }
             }
         }
     }
@@ -1880,7 +1877,7 @@ else if (req.close() && req.initiatingNodeId().equals(ctx.localNodeId())) {
      * @param req Request to complete future for.
      * @param err Error to be passed to futures.
      */
-    public void completeStartFuture(DynamicCacheChangeRequest req, @Nullable Throwable err) {
+    void completeStartFuture(DynamicCacheChangeRequest req, @Nullable Throwable err) {
         DynamicCacheStartFuture fut = (DynamicCacheStartFuture)pendingFuts.get(maskNull(req.cacheName()));
 
         assert req.deploymentId() != null;
@@ -2648,6 +2645,8 @@ private boolean onCacheChangeRequested(DynamicCacheChangeBatch batch,
 
         boolean incMinorTopVer = false;
 
+        ExchangeActions exchangeActions = new ExchangeActions();
+
         for (DynamicCacheChangeRequest req : batch.requests()) {
             initReceivedCacheConfiguration(req);
 
@@ -2729,6 +2728,8 @@ private boolean onCacheChangeRequested(DynamicCacheChangeBatch batch,
                             req.nearCacheConfiguration() != null);
 
                         needExchange = true;
+
+                        exchangeActions.addCacheToStart(req, startDesc);
                     }
                 }
                 else {
@@ -2755,15 +2756,19 @@ private boolean onCacheChangeRequested(DynamicCacheChangeBatch batch,
                             needExchange = clientReq && ctx.discovery().addClientNode(req.cacheName(),
                                 req.initiatingNodeId(),
                                 req.nearCacheConfiguration() != null);
-
-                            if (needExchange)
-                                req.clientStartOnly(true);
                         }
                     }
+
+                    if (needExchange) {
+                        req.clientStartOnly(true);
+
+                        exchangeActions.addCacheToStart(req, desc);
+                    }
                 }
 
                 if (!needExchange && desc != null)
                     req.cacheFutureTopologyVersion(desc.startTopologyVersion());
+
             }
             else {
                 assert req.stop() ^ req.close() : req;
@@ -2777,11 +2782,16 @@ private boolean onCacheChangeRequested(DynamicCacheChangeBatch batch,
                         ctx.discovery().removeCacheFilter(req.cacheName());
 
                         needExchange = true;
+
+                        exchangeActions.addCacheToStop(req, desc);
                     }
                     else {
                         assert req.close() : req;
 
                         needExchange = ctx.discovery().onClientCacheClose(req.cacheName(), req.initiatingNodeId());
+
+                        if (needExchange)
+                            exchangeActions.addCacheToClose(req, desc);
                     }
                 }
             }
@@ -2791,6 +2801,12 @@ private boolean onCacheChangeRequested(DynamicCacheChangeBatch batch,
             incMinorTopVer |= needExchange;
         }
 
+        if (incMinorTopVer) {
+            assert !exchangeActions.empty() : exchangeActions;
+
+            batch.exchangeActions(exchangeActions);
+        }
+
         return incMinorTopVer;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index 4377c6e4aa6a8..79970f4f29ede 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -26,9 +26,9 @@
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridNodeOrderComparator;
+import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -37,7 +37,6 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
@@ -73,20 +72,20 @@ public class GridDhtAssignmentFetchFuture extends GridFutureAdapter<GridDhtAffin
 
     /**
      * @param ctx Context.
-     * @param cacheName Cache name.
+     * @param cacheId Cache ID.
      * @param topVer Topology version.
      * @param discoCache Discovery cache.
      */
     public GridDhtAssignmentFetchFuture(
         GridCacheSharedContext ctx,
-        String cacheName,
+        Integer cacheId,
         AffinityTopologyVersion topVer,
         DiscoCache discoCache
     ) {
         this.ctx = ctx;
-        this.key = new T2<>(CU.cacheId(cacheName), topVer);
+        this.key = new T2<>(cacheId, topVer);
 
-        Collection<ClusterNode> availableNodes = discoCache.cacheAffinityNodes(CU.cacheId(cacheName));
+        Collection<ClusterNode> availableNodes = discoCache.cacheAffinityNodes(cacheId);
 
         LinkedList<ClusterNode> tmp = new LinkedList<>();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 5bbcfc8d9ae80..0de932780a1ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -54,6 +54,7 @@
 import org.apache.ignite.internal.processors.cache.DynamicCacheChangeFailureMessage;
 import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.ExchangeActions;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
@@ -160,8 +161,7 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     private AtomicReference<GridCacheVersion> lastVer = new AtomicReference<>();
 
     /**
-     * Messages received on non-coordinator are stored in case if this node
-     * becomes coordinator.
+     * Messages received on non-coordinator are stored in case if this node becomes coordinator.
      */
     private final Map<ClusterNode, GridDhtPartitionsSingleMessage> singleMsgs = new ConcurrentHashMap8<>();
 
@@ -179,9 +179,6 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** Logger. */
     private IgniteLogger log;
 
-    /** Dynamic cache change requests. */
-    private Collection<DynamicCacheChangeRequest> reqs;
-
     /** */
     private CacheAffinityChangeMessage affChangeMsg;
 
@@ -212,9 +209,11 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** Forced Rebalance future. */
     private GridCompoundFuture<Boolean, Boolean> forcedRebFut;
 
+    /** Cache change requests. */
+    private ExchangeActions exchActions;
+
     /**
-     * Dummy future created to trigger reassignments if partition
-     * topology changed while preloading.
+     * Dummy future created to trigger reassignments if partition topology changed while preloading.
      *
      * @param cctx Cache context.
      * @param reassign Dummy reassign flag.
@@ -239,8 +238,7 @@ public GridDhtPartitionsExchangeFuture(
     }
 
     /**
-     * Force preload future created to trigger reassignments if partition
-     * topology changed while preloading.
+     * Force preload future created to trigger reassignments if partition topology changed while preloading.
      *
      * @param cctx Cache context.
      * @param discoEvt Discovery event.
@@ -266,14 +264,14 @@ public GridDhtPartitionsExchangeFuture(GridCacheSharedContext cctx, DiscoveryEve
      * @param cctx Cache context.
      * @param busyLock Busy lock.
      * @param exchId Exchange ID.
-     * @param reqs Cache change requests.
+     * @param exchActions Exchange actions.
      * @param affChangeMsg Affinity change message.
      */
     public GridDhtPartitionsExchangeFuture(
         GridCacheSharedContext cctx,
         ReadWriteLock busyLock,
         GridDhtPartitionExchangeId exchId,
-        Collection<DynamicCacheChangeRequest> reqs,
+        ExchangeActions exchActions,
         CacheAffinityChangeMessage affChangeMsg
     ) {
         assert busyLock != null;
@@ -286,7 +284,7 @@ public GridDhtPartitionsExchangeFuture(
         this.cctx = cctx;
         this.busyLock = busyLock;
         this.exchId = exchId;
-        this.reqs = reqs;
+        this.exchActions = exchActions;
         this.affChangeMsg = affChangeMsg;
 
         log = cctx.logger(getClass());
@@ -297,13 +295,6 @@ public GridDhtPartitionsExchangeFuture(
             log.debug("Creating exchange future [localNode=" + cctx.localNodeId() + ", fut=" + this + ']');
     }
 
-    /**
-     * @param reqs Cache change requests.
-     */
-    public void cacheChangeRequests(Collection<DynamicCacheChangeRequest> reqs) {
-        this.reqs = reqs;
-    }
-
     /**
      * @param affChangeMsg Affinity change message.
      */
@@ -377,16 +368,7 @@ public boolean isCacheAdded(int cacheId, AffinityTopologyVersion topVer) {
      * @return {@code True} if non-client cache was added during this exchange.
      */
     public boolean cacheStarted(int cacheId) {
-        if (!F.isEmpty(reqs)) {
-            for (DynamicCacheChangeRequest req : reqs) {
-                if (req.start() && !req.clientStartOnly()) {
-                    if (CU.cacheId(req.cacheName()) == cacheId)
-                        return true;
-                }
-            }
-        }
-
-        return false;
+        return exchActions != null && exchActions.cacheStarted(cacheId);
     }
 
     /**
@@ -486,8 +468,11 @@ public void init() throws IgniteInterruptedCheckedException {
             Collection<DynamicCacheDescriptor> receivedCaches;
 
             if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
-                if (!F.isEmpty(reqs))
+                if (exchActions != null) {
+                    assert !exchActions.empty();
+
                     exchange = onCacheChangeRequest(crdNode);
+                }
                 else {
                     assert affChangeMsg != null : this;
 
@@ -603,7 +588,7 @@ public void init() throws IgniteInterruptedCheckedException {
     }
 
     /**
-      * @throws IgniteCheckedException If failed.
+     * @throws IgniteCheckedException If failed.
      */
     private void initTopologies() throws IgniteCheckedException {
         if (crd != null) {
@@ -648,29 +633,28 @@ private void updateTopologies(boolean crd) throws IgniteCheckedException {
 
     /**
      * @param crd Coordinator flag.
-     * @throws IgniteCheckedException If failed.
      * @return Exchange type.
+     * @throws IgniteCheckedException If failed.
      */
     private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedException {
-        assert !F.isEmpty(reqs) : this;
+        assert exchActions != null;
 
-        boolean clientOnly = cctx.affinity().onCacheChangeRequest(this, crd, reqs);
+        boolean clientOnly = cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
 
         if (clientOnly) {
             boolean clientCacheStarted = false;
 
-            for (DynamicCacheChangeRequest req : reqs) {
-                if (req.start() && req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId())) {
+            for (ExchangeActions.CacheActionData action :exchActions.cacheStartRequests()){
+                DynamicCacheChangeRequest req = action.request();
+
+                if (req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId())) {
                     clientCacheStarted = true;
 
                     break;
                 }
             }
 
-            if (clientCacheStarted)
-                return ExchangeType.CLIENT;
-            else
-                return ExchangeType.NONE;
+            return clientCacheStarted ? ExchangeType.CLIENT : ExchangeType.NONE;
         }
         else
             return cctx.kernalContext().clientNode() ? ExchangeType.CLIENT : ExchangeType.ALL;
@@ -678,8 +662,8 @@ private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedExcep
 
     /**
      * @param crd Coordinator flag.
-     * @throws IgniteCheckedException If failed.
      * @return Exchange type.
+     * @throws IgniteCheckedException If failed.
      */
     private ExchangeType onAffinityChangeRequest(boolean crd) throws IgniteCheckedException {
         assert affChangeMsg != null : this;
@@ -694,8 +678,8 @@ private ExchangeType onAffinityChangeRequest(boolean crd) throws IgniteCheckedEx
 
     /**
      * @param crd Coordinator flag.
-     * @throws IgniteCheckedException If failed.
      * @return Exchange type.
+     * @throws IgniteCheckedException If failed.
      */
     private ExchangeType onClientNodeEvent(boolean crd) throws IgniteCheckedException {
         assert CU.clientNode(discoEvt.eventNode()) : this;
@@ -718,8 +702,8 @@ private ExchangeType onClientNodeEvent(boolean crd) throws IgniteCheckedExceptio
 
     /**
      * @param crd Coordinator flag.
-     * @throws IgniteCheckedException If failed.
      * @return Exchange type.
+     * @throws IgniteCheckedException If failed.
      */
     private ExchangeType onServerNodeEvent(boolean crd) throws IgniteCheckedException {
         assert !CU.clientNode(discoEvt.eventNode()) : this;
@@ -1003,19 +987,7 @@ private void dumpPendingObjects() {
      * @return {@code True} if cache is stopping by this exchange.
      */
     public boolean stopping(int cacheId) {
-        boolean stopping = false;
-
-        if (!F.isEmpty(reqs)) {
-            for (DynamicCacheChangeRequest req : reqs) {
-                if (cacheId == CU.cacheId(req.cacheName())) {
-                    stopping = req.stop();
-
-                    break;
-                }
-            }
-        }
-
-        return stopping;
+        return exchActions != null && exchActions.cacheStopped(cacheId);
     }
 
     /**
@@ -1150,7 +1122,10 @@ private void sendPartitions(ClusterNode oldestNode) {
 
         cctx.exchange().onExchangeDone(this, err);
 
-        cctx.cache().onExchangeDone(exchId.topologyVersion(), reqs, err);
+        cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err);
+
+        if (exchActions != null)
+            exchActions.completeRequestFutures(cctx, err);
 
         if (super.onDone(res, err) && realExchange) {
             if (log.isDebugEnabled())
@@ -1164,7 +1139,7 @@ private void sendPartitions(ClusterNode oldestNode) {
                     cacheCtx.config().getAffinity().removeNode(exchId.nodeId());
             }
 
-            reqs = null;
+            exchActions = null;
 
             if (discoEvt instanceof DiscoveryCustomEvent)
                 ((DiscoveryCustomEvent)discoEvt).customMessage(null);
@@ -1358,8 +1333,8 @@ private void onAffinityInitialized(IgniteInternalFuture<Map<Integer, Map<Integer
     }
 
     /**
-     * Creates an IgniteCheckedException that is used as root cause of the exchange initialization failure.
-     * This method aggregates all the exceptions provided from all participating nodes.
+     * Creates an IgniteCheckedException that is used as root cause of the exchange initialization failure. This method
+     * aggregates all the exceptions provided from all participating nodes.
      *
      * @param globalExceptions collection exceptions from all participating nodes.
      * @return exception that represents a cause of the exchange initialization failure.
@@ -1387,45 +1362,33 @@ private IgniteCheckedException createExchangeException(Map<UUID, Exception> glob
      * @return {@code true} if the given {@code discoEvt} supports the rollback procedure.
      */
     private boolean isRollbackSupported() {
-        boolean rollbackSupported = false;
-
         if (!discoCache.checkAttribute(ATTR_EXCHANGE_ROLLBACK_SUPPORTED, Boolean.TRUE))
             return false;
 
         // Currently the rollback process is supported for dynamically started caches.
-        if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && !F.isEmpty(reqs)) {
-            for (DynamicCacheChangeRequest req : reqs) {
-                if (req.start()) {
-                    rollbackSupported = true;
-
-                    break;
-                }
-            }
-        }
-
-        return rollbackSupported;
+        return discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && exchActions != null && exchActions.hasStart();
     }
 
     /**
-     * Tries to revert all the changes that were done during initialization phase
-     * in case of the given {@code discoEvt} supports the rollback procedure.
+     * Tries to revert all the changes that were done during initialization phase in case of the given {@code discoEvt}
+     * supports the rollback procedure.
      */
     private void rollbackExchange() {
-        if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && !F.isEmpty(reqs)) {
-            for (DynamicCacheChangeRequest req : reqs) {
-                if (req.start()) {
-                    DynamicCacheChangeRequest stopReq =
-                        new DynamicCacheChangeRequest(req.cacheName(), cctx.localNodeId());
+        if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && exchActions != null) {
+            for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) {
+                DynamicCacheChangeRequest req = action.request();
 
-                    stopReq.stop(true);
-                    stopReq.deploymentId(req.deploymentId());
+                DynamicCacheChangeRequest stopReq =
+                    new DynamicCacheChangeRequest(req.cacheName(), cctx.localNodeId());
 
-                    // Cleanup GridCacheProcessor.
-                    cctx.cache().forceCloseCache(stopReq);
+                stopReq.stop(true);
+                stopReq.deploymentId(req.deploymentId());
 
-                    // Cleanup CacheAffinitySharedManager.
-                    cctx.affinity().forceCloseCache(this, crd.isLocal(), Collections.singletonList(stopReq));
-                }
+                // Cleanup GridCacheProcessor.
+                cctx.cache().forceCloseCache(stopReq);
+
+                // Cleanup CacheAffinitySharedManager.
+                cctx.affinity().forceCloseCache(this, crd.isLocal(), Collections.singletonList(stopReq));
             }
         }
     }
@@ -1452,10 +1415,11 @@ private void onAllReceived() {
             if (!F.isEmpty(exchangeGlobalExceptions) && isRollbackSupported()) {
                 IgniteCheckedException err = createExchangeException(exchangeGlobalExceptions);
 
-                List<String> cacheNames = new ArrayList<>(reqs.size());
+                List<String> cacheNames = new ArrayList<>();
 
-                for (DynamicCacheChangeRequest req : reqs)
-                    cacheNames.add(req.cacheName());
+                for (ExchangeActions.CacheActionData action : F.concat(false,
+                    exchActions.cacheStartRequests(), exchActions.cacheStopRequests()))
+                    cacheNames.add(action.request().cacheName());
 
                 DynamicCacheChangeFailureMessage msg = new DynamicCacheChangeFailureMessage(
                     cctx.localNode(), exchId, err, cacheNames);
@@ -1621,7 +1585,7 @@ private void processMessage(ClusterNode node, GridDhtPartitionsFullMessage msg)
             if (!crd.equals(node)) {
                 if (log.isDebugEnabled())
                     log.debug("Received full partition map from unexpected node [oldest=" + crd.id() +
-                            ", nodeId=" + node.id() + ']');
+                        ", nodeId=" + node.id() + ']');
 
                 if (node.order() > crd.order())
                     fullMsgs.put(node, msg);
@@ -1950,6 +1914,16 @@ public boolean reconnectOnError(Throwable e) {
         return exchId.compareTo(fut.exchId);
     }
 
+    /**
+     * @param exchActions New cache change requests.
+     */
+    public void exchangeActions(ExchangeActions exchActions) {
+        assert exchActions == null || !exchActions.empty() : exchActions;
+        assert evtLatch != null && evtLatch.getCount() == 1L : this;
+
+        this.exchActions = exchActions;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean equals(Object o) {
         if (this == o)

From ec6c1194f941c7818656a777d9fe99bb70db58d6 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 15 Sep 2017 14:54:07 +0300
Subject: [PATCH 384/516] ignite-3495: metrics for atomic caches

---
 .../processors/cache/GridCacheAdapter.java    | 258 ++++------
 .../distributed/dht/GridDhtCacheAdapter.java  |  26 +
 .../dht/GridPartitionedGetFuture.java         |  21 +-
 .../dht/GridPartitionedSingleGetFuture.java   |   8 +
 .../dht/atomic/GridDhtAtomicCache.java        |  33 +-
 .../GridNearAtomicAbstractUpdateFuture.java   |  25 +
 .../atomic/GridNearAtomicUpdateFuture.java    |  25 +
 .../dht/colocated/GridDhtColocatedCache.java  |  19 +-
 .../distributed/near/GridNearGetFuture.java   |   8 +
 .../cache/local/GridLocalCache.java           |  42 ++
 .../local/atomic/GridLocalAtomicCache.java    |  78 ++-
 ...acheAtomicClientServerMetricsSelfTest.java | 452 ++++++++++++++++++
 .../IgniteCacheMetricsSelfTestSuite.java      |   4 +-
 13 files changed, 816 insertions(+), 183 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicClientServerMetricsSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index ff1850dbcfe39..74415d2733f43 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -1406,10 +1406,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Nullable @Override public V get(K key) throws IgniteCheckedException {
         A.notNull(key, "key");
 
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         boolean keepBinary = ctx.keepBinary();
 
         if (keepBinary)
@@ -1423,9 +1419,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
             val = (V)ctx.config().getInterceptor().onGet(key, val);
         }
 
-        if (statsEnabled)
-            metrics0().addGetTimeNanos(System.nanoTime() - start);
-
         return val;
     }
 
@@ -1433,10 +1426,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Nullable @Override public CacheEntry<K, V> getEntry(K key) throws IgniteCheckedException {
         A.notNull(key, "key");
 
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         boolean keepBinary = ctx.keepBinary();
 
         if (keepBinary)
@@ -1459,9 +1448,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
             val = (val0 != null) ? new CacheEntryImplEx<>(key, val0, t != null ? t.version() : null) : null;
         }
 
-        if (statsEnabled)
-            metrics0().addGetTimeNanos(System.nanoTime() - start);
-
         return val;
     }
 
@@ -1469,10 +1455,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Override public IgniteInternalFuture<V> getAsync(final K key) {
         A.notNull(key, "key");
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         final boolean keepBinary = ctx.keepBinary();
 
         final K key0 = keepBinary ? (K)ctx.toCacheKeyObject(key) : key;
@@ -1488,9 +1470,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
                 }
             });
 
-        if (statsEnabled)
-            fut.listen(new UpdateGetTimeStatClosure<V>(metrics0(), start));
-
         return fut;
     }
 
@@ -1498,10 +1477,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Override public IgniteInternalFuture<CacheEntry<K, V>> getEntryAsync(final K key) {
         A.notNull(key, "key");
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         final boolean keepBinary = ctx.keepBinary();
 
         final K key0 = keepBinary ? (K)ctx.toCacheKeyObject(key) : key;
@@ -1535,9 +1510,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
             }
         });
 
-        if (statsEnabled)
-            fut.listen(new UpdateGetTimeStatClosure<EntryGetResult>(metrics0(), start));
-
         return fr;
     }
 
@@ -1545,18 +1517,11 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Override public final Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         Map<K, V> map = getAll0(keys, !ctx.keepBinary(), false);
 
         if (ctx.config().getInterceptor() != null)
             map = interceptGet(keys, map);
 
-        if (statsEnabled)
-            metrics0().addGetTimeNanos(System.nanoTime() - start);
-
         return map;
     }
 
@@ -1565,22 +1530,16 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
         throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         Map<K, EntryGetResult> map = (Map<K, EntryGetResult>)getAll0(keys, !ctx.keepBinary(), true);
 
         Collection<CacheEntry<K, V>> res = new HashSet<>();
 
         if (ctx.config().getInterceptor() != null)
             res = interceptGetEntries(keys, map);
-        else
+        else {
             for (Map.Entry<K, EntryGetResult> e : map.entrySet())
                 res.add(new CacheEntryImplEx<>(e.getKey(), (V)e.getValue().value(), e.getValue().version()));
-
-        if (statsEnabled)
-            metrics0().addGetTimeNanos(System.nanoTime() - start);
+        }
 
         return res;
     }
@@ -1589,10 +1548,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
     @Override public IgniteInternalFuture<Map<K, V>> getAllAsync(@Nullable final Collection<? extends K> keys) {
         A.notNull(keys, "keys");
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         IgniteInternalFuture<Map<K, V>> fut = getAllAsync(keys, !ctx.keepBinary(), false);
 
         if (ctx.config().getInterceptor() != null)
@@ -1602,9 +1557,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
                 }
             });
 
-        if (statsEnabled)
-            fut.listen(new UpdateGetTimeStatClosure<Map<K, V>>(metrics0(), start));
-
         return fut;
     }
 
@@ -1613,10 +1565,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
         @Nullable final Collection<? extends K> keys) {
         A.notNull(keys, "keys");
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         IgniteInternalFuture<Map<K, EntryGetResult>> fut =
             (IgniteInternalFuture<Map<K, EntryGetResult>>)
                 ((IgniteInternalFuture)getAllAsync(keys, !ctx.keepBinary(), true));
@@ -1641,9 +1589,6 @@ public final V getTopologySafe(K key) throws IgniteCheckedException {
                 }
             });
 
-        if (statsEnabled)
-            fut.listen(new UpdateGetTimeStatClosure<Map<K, EntryGetResult>>(metrics0(), start));
-
         return rf;
     }
 
@@ -2212,21 +2157,12 @@ private void clearReservationsIfNeeded(
      */
     @Nullable public V getAndPut(final K key, final V val, @Nullable final CacheEntryPredicate filter)
         throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key", val, "val");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        V prevVal = getAndPut0(key, val, filter);
-
-        if (statsEnabled)
-            metrics0().addPutAndGetTimeNanos(System.nanoTime() - start);
-
-        return prevVal;
+        return getAndPut0(key, val, filter);
     }
 
     /**
@@ -2261,21 +2197,12 @@ protected V getAndPut0(final K key, final V val, @Nullable final CacheEntryPredi
      * @return Put operation future.
      */
     protected final IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key", val, "val");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<V> fut = getAndPutAsync0(key, val, filter);
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutAndGetTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
+        return getAndPutAsync0(key, val, filter);
     }
 
     /**
@@ -2288,7 +2215,11 @@ public IgniteInternalFuture<V> getAndPutAsync0(final K key,
         final V val,
         @Nullable final CacheEntryPredicate filter)
     {
-        return asyncOp(new AsyncOp<V>() {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
             @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAsync(ctx, readyTopVer, key, val, true, filter)
                     .chain((IgniteClosure<IgniteInternalFuture<GridCacheReturn>, V>)RET2VAL);
@@ -2298,6 +2229,11 @@ public IgniteInternalFuture<V> getAndPutAsync0(final K key,
                 return "putAsync [key=" + key + ", val=" + val + ", filter=" + filter + ']';
             }
         });
+
+        if (statsEnabled)
+            fut.listen(new UpdatePutAndGetTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -2316,21 +2252,12 @@ public IgniteInternalFuture<V> getAndPutAsync0(final K key,
      */
     public boolean put(final K key, final V val, final CacheEntryPredicate filter)
         throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key", val, "val");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        boolean stored = put0(key, val, filter);
-
-        if (statsEnabled && stored)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
-
-        return stored;
+        return put0(key, val, filter);
     }
 
     /**
@@ -2344,6 +2271,10 @@ public boolean put(final K key, final V val, final CacheEntryPredicate filter)
      */
     protected boolean put0(final K key, final V val, final CacheEntryPredicate filter)
         throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         Boolean res = syncOp(new SyncOp<Boolean>(true) {
             @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return tx.putAsync(ctx, null, key, val, false, filter).get().success();
@@ -2356,6 +2287,9 @@ protected boolean put0(final K key, final V val, final CacheEntryPredicate filte
 
         assert res != null;
 
+        if (statsEnabled && Boolean.TRUE.equals(res))
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
+
         return res;
     }
 
@@ -2647,16 +2581,7 @@ public final IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable Cach
         if (keyCheck)
             validateCacheKey(key);
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
-        IgniteInternalFuture<Boolean> fut = putAsync0(key, val, filter);
-
-        if (statsEnabled)
-            fut.listen(new UpdatePutTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+        return putAsync0(key, val, filter);
     }
 
     /**
@@ -2665,9 +2590,12 @@ public final IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable Cach
      * @param filter Optional filter.
      * @return Putx operation future.
      */
-    public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
-        @Nullable final CacheEntryPredicate filter) {
-        return asyncOp(new AsyncOp<Boolean>() {
+    public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val, @Nullable final CacheEntryPredicate filter) {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
             @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.putAsync(ctx,
                     readyTopVer,
@@ -2685,6 +2613,11 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
                     "filter", filter, false);
             }
         });
+
+        if (statsEnabled)
+            fut.listen(new UpdatePutTimeStatClosure<Boolean>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -2754,17 +2687,10 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
         if (F.isEmpty(m))
             return;
 
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         if (keyCheck)
             validateCacheKeys(m.keySet());
 
         putAll0(m);
-
-        if (statsEnabled)
-            metrics0().addPutTimeNanos(System.nanoTime() - start);
     }
 
     /**
@@ -2772,6 +2698,10 @@ public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
      * @throws IgniteCheckedException If failed.
      */
     protected void putAll0(final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
         syncOp(new SyncInOp(m.size() == 1) {
             @Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 tx.putAllAsync(ctx, null, m, false).get();
@@ -2781,6 +2711,9 @@ protected void putAll0(final Map<? extends K, ? extends V> m) throws IgniteCheck
                 return "putAll [map=" + m + ']';
             }
         });
+
+        if (statsEnabled)
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
@@ -2815,21 +2748,12 @@ protected IgniteInternalFuture<?> putAllAsync0(final Map<? extends K, ? extends
 
     /** {@inheritDoc} */
     @Nullable @Override public V getAndRemove(final K key) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        V prevVal = getAndRemove0(key);
-
-        if (statsEnabled)
-            metrics0().addRemoveAndGetTimeNanos(System.nanoTime() - start);
-
-        return prevVal;
+        return getAndRemove0(key);
     }
 
     /**
@@ -2868,21 +2792,12 @@ protected V getAndRemove0(final K key) throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<V> getAndRemoveAsync(final K key) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<V> fut = getAndRemoveAsync0(key);
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
-
-        return fut;
+        return getAndRemoveAsync0(key);
     }
 
     /**
@@ -2890,7 +2805,11 @@ protected V getAndRemove0(final K key) throws IgniteCheckedException {
      * @return Future.
      */
     protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
-        return asyncOp(new AsyncOp<V>() {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
             @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 // TODO should we invoke interceptor here?
                 return tx.removeAllAsync(ctx,
@@ -2905,6 +2824,11 @@ protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
                 return "removeAsync [key=" + key + ']';
             }
         });
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -2925,10 +2849,6 @@ protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
 
     /** {@inheritDoc} */
     @Override public void removeAll(final Collection<? extends K> keys) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(keys, "keys");
 
         if (F.isEmpty(keys))
@@ -2938,9 +2858,6 @@ protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
             validateCacheKeys(keys);
 
         removeAll0(keys);
-
-        if (statsEnabled)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
     }
 
     /**
@@ -2948,6 +2865,10 @@ protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
      * @throws IgniteCheckedException If failed.
      */
     protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
         syncOp(new SyncInOp(keys.size() == 1) {
             @Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 tx.removeAllAsync(ctx,
@@ -2962,6 +2883,9 @@ protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheck
                 return "removeAll [keys=" + keys + ']';
             }
         });
+
+        if (statsEnabled)
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
     }
 
     /** {@inheritDoc} */
@@ -2969,19 +2893,10 @@ protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheck
         if (F.isEmpty(keys))
             return new GridFinishedFuture<Object>();
 
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         if (keyCheck)
             validateCacheKeys(keys);
 
-        IgniteInternalFuture<Object> fut = removeAllAsync0(keys);
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
-
-        return fut;
+        return removeAllAsync0(keys);
     }
 
     /**
@@ -2989,7 +2904,11 @@ protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheck
      * @return Future.
      */
     protected IgniteInternalFuture<Object> removeAllAsync0(final Collection<? extends K> keys) {
-        return asyncOp(new AsyncOp(keys) {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<Object> fut = asyncOp(new AsyncOp(keys) {
             @Override public IgniteInternalFuture<?> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.removeAllAsync(ctx,
                     readyTopVer,
@@ -3003,6 +2922,11 @@ protected IgniteInternalFuture<Object> removeAllAsync0(final Collection<? extend
                 return "removeAllAsync [keys=" + keys + ']';
             }
         });
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -3017,21 +2941,12 @@ protected IgniteInternalFuture<Object> removeAllAsync0(final Collection<? extend
      * @throws IgniteCheckedException If failed.
      */
     public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
-        boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        boolean rmv = remove0(key, filter);
-
-        if (statsEnabled && rmv)
-            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
-
-        return rmv;
+        return remove0(key, filter);
     }
 
     /**
@@ -3041,6 +2956,10 @@ public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws
      * @throws IgniteCheckedException If failed.
      */
     protected boolean remove0(final K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
         Boolean res = syncOp(new SyncOp<Boolean>(true) {
             @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
                 return tx.removeAllAsync(ctx,
@@ -3058,6 +2977,9 @@ protected boolean remove0(final K key, final CacheEntryPredicate filter) throws
 
         assert res != null;
 
+        if (statsEnabled && Boolean.TRUE.equals(res))
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+
         return res;
     }
 
@@ -3074,21 +2996,12 @@ protected boolean remove0(final K key, final CacheEntryPredicate filter) throws
      * @return Putx operation future.
      */
     public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final CacheEntryPredicate filter) {
-        final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
-        final long start = statsEnabled ? System.nanoTime() : 0L;
-
         A.notNull(key, "key");
 
         if (keyCheck)
             validateCacheKey(key);
 
-        IgniteInternalFuture<Boolean> fut = removeAsync0(key, filter);
-
-        if (statsEnabled)
-            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
-
-        return fut;
+        return removeAsync0(key, filter);
     }
 
     /**
@@ -3097,7 +3010,11 @@ public IgniteInternalFuture<Boolean> removeAsync(final K key, @Nullable final Ca
      * @return Future.
      */
     protected IgniteInternalFuture<Boolean> removeAsync0(final K key, @Nullable final CacheEntryPredicate filter) {
-        return asyncOp(new AsyncOp<Boolean>() {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
             @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                 return tx.removeAllAsync(ctx,
                     readyTopVer,
@@ -3112,6 +3029,11 @@ protected IgniteInternalFuture<Boolean> removeAsync0(final K key, @Nullable fina
                 return "removeAsync [key=" + key + ", filter=" + filter + ']';
             }
         });
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 417eb3536536f..ac73907bdf175 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -761,6 +761,10 @@ private IgniteInternalFuture<GridCacheEntryInfo> getDhtSingleAsync(
     protected void processNearSingleGetRequest(final UUID nodeId, final GridNearSingleGetRequest req) {
         assert ctx.affinityNode();
 
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        final long start = statsEnabled ? System.nanoTime() : 0L;
+
         final CacheExpiryPolicy expiryPlc = CacheExpiryPolicy.fromRemote(req.createTtl(), req.accessTtl());
 
         IgniteInternalFuture<GridCacheEntryInfo> fut =
@@ -851,6 +855,15 @@ else if (req.needVersion())
                 sendTtlUpdateRequest(expiryPlc);
             }
         });
+
+        if (statsEnabled) {
+            fut.listen(new CI1<IgniteInternalFuture<GridCacheEntryInfo>>() {
+                @Override public void apply(IgniteInternalFuture<GridCacheEntryInfo> f) {
+                    if (!req.skipValues() && f.error() == null)
+                        metrics0().addGetTimeNanos(System.nanoTime() - start);
+                }
+            });
+        }
     }
 
     /**
@@ -861,6 +874,10 @@ protected void processNearGetRequest(final UUID nodeId, final GridNearGetRequest
         assert ctx.affinityNode();
         assert !req.reload() : req;
 
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        final long start = statsEnabled ? System.nanoTime() : 0L;
+
         final CacheExpiryPolicy expiryPlc = CacheExpiryPolicy.fromRemote(req.createTtl(), req.accessTtl());
 
         IgniteInternalFuture<Collection<GridCacheEntryInfo>> fut =
@@ -915,6 +932,15 @@ protected void processNearGetRequest(final UUID nodeId, final GridNearGetRequest
                 sendTtlUpdateRequest(expiryPlc);
             }
         });
+
+        if (statsEnabled) {
+            fut.listen(new CI1<IgniteInternalFuture<Collection<GridCacheEntryInfo>>>() {
+                @Override public void apply(IgniteInternalFuture<Collection<GridCacheEntryInfo>> f) {
+                    if (!req.skipValues() && f.error() == null)
+                        metrics0().addGetTimeNanos(System.nanoTime() - start);
+                }
+            });
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 519239aecb532..9b7c8dc8de284 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -53,7 +53,6 @@
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -281,6 +280,10 @@ private void map(
 
             // If this is the primary or backup node for the keys.
             if (n.isLocal()) {
+                final boolean statsEnabled = cctx.config().isStatisticsEnabled();
+
+                final long start = (statsEnabled)? System.nanoTime() : 0L;
+
                 final GridDhtFuture<Collection<GridCacheEntryInfo>> fut =
                     cache().getDhtAsync(n.id(),
                         -1,
@@ -316,7 +319,12 @@ private void map(
                 add(fut.chain(new C1<IgniteInternalFuture<Collection<GridCacheEntryInfo>>, Map<K, V>>() {
                     @Override public Map<K, V> apply(IgniteInternalFuture<Collection<GridCacheEntryInfo>> fut) {
                         try {
-                            return createResultMap(fut.get());
+                            Map<K, V> result = createResultMap(fut.get());
+
+                            if (!skipVals && statsEnabled && fut.error() == null)
+                                cctx.cache().metrics0().addGetTimeNanos(System.nanoTime() - start);
+
+                            return result;
                         }
                         catch (Exception e) {
                             U.error(log, "Failed to get values from dht cache [fut=" + fut + "]", e);
@@ -435,6 +443,10 @@ private boolean map(
     private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
         assert cctx.affinityNode() : this;
 
+        boolean statsEnabled = cctx.config().isStatisticsEnabled();
+
+        long start = (statsEnabled)? System.nanoTime() : 0L;
+
         GridDhtCacheAdapter<K, V> cache = cache();
 
         while (true) {
@@ -516,9 +528,12 @@ private boolean localGet(KeyCacheObject key, int part, Map<K, V> locVals) {
 
                 // Entry not found, do not continue search if topology did not change and there is no store.
                 if (!cctx.readThroughConfigured() && (topStable || partitionOwned(part))) {
-                    if (!skipVals && cctx.config().isStatisticsEnabled())
+                    if (!skipVals && statsEnabled) {
                         cache.metrics0().onRead(false);
 
+                        cache.metrics0().addGetTimeNanos(System.nanoTime() - start);
+                    }
+
                     return true;
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index a3f6b72376e8f..25e9f0042672a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -122,6 +122,9 @@ public class GridPartitionedSingleGetFuture extends GridFutureAdapter<Object> im
     /** */
     private final boolean keepCacheObjects;
 
+    /** Future start time in nanoseconds. */
+    private final long startTimeNanos;
+
     /** */
     @GridToStringInclude
     private ClusterNode node;
@@ -182,6 +185,8 @@ public GridPartitionedSingleGetFuture(
 
         futId = IgniteUuid.randomUuid();
 
+        startTimeNanos = cctx.config().isStatisticsEnabled() ? System.nanoTime() : 0L;
+
         if (log == null)
             log = U.logger(cctx.kernalContext(), logRef, GridPartitionedSingleGetFuture.class);
     }
@@ -735,6 +740,9 @@ private void remap(final AffinityTopologyVersion topVer) {
 
             cctx.dht().sendTtlUpdateRequest(expiryPlc);
 
+            if (cctx.config().isStatisticsEnabled() && !trackable && !skipVals && err == null)
+                cctx.cache().metrics0().addGetTimeNanos(System.nanoTime() - startTimeNanos);
+
             return true;
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index ffe68ed2cfee7..527b4a7d80926 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1573,6 +1573,10 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
 
         // Optimisation: try to resolve value locally and escape 'get future' creation.
         if (!forcePrimary && ctx.affinityNode()) {
+            boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+            long start = (statsEnabled)? System.nanoTime() : 0L;
+
             Map<K, V> locVals = U.newHashMap(keys.size());
 
             boolean success = true;
@@ -1599,7 +1603,7 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
                                     null,
                                     /*swap*/true,
                                     /*unmarshal*/true,
-                                    /**update-metrics*/false,
+                                    /*update-metrics*/false,
                                     /*event*/!skipVals,
                                     subjId,
                                     null,
@@ -1618,7 +1622,7 @@ private IgniteInternalFuture<Map<K, V>> getAllAsync0(@Nullable Collection<KeyCac
                                     null,
                                     /*swap*/true,
                                     /*read-through*/false,
-                                    /**update-metrics*/false,
+                                    /*update-metrics*/false,
                                     /*event*/!skipVals,
                                     /*temporary*/false,
                                     subjId,
@@ -1672,6 +1676,9 @@ else if (!skipVals && ctx.config().isStatisticsEnabled())
             if (success) {
                 sendTtlUpdateRequest(expiry);
 
+                if (!skipVals && statsEnabled)
+                    metrics0().addGetTimeNanos(System.nanoTime() - start);
+
                 return new GridFinishedFuture<>(locVals);
             }
         }
@@ -3203,9 +3210,31 @@ private void processNearAtomicUpdateRequest(UUID nodeId, GridNearAtomicAbstractU
                 ", node=" + nodeId + ']');
         }
 
+        long start = (ctx.config().isStatisticsEnabled())? System.nanoTime() : 0L;
+
         req.nodeId(ctx.localNodeId());
 
         updateAllAsyncInternal(nodeId, req, updateReplyClos);
+
+        if (ctx.config().isStatisticsEnabled()) {
+            boolean retVal = req.returnValue();
+
+            long duration = System.nanoTime() - start;
+
+            if (req.operation() == UPDATE) {
+                if (retVal)
+                    metrics0().addPutAndGetTimeNanos(duration);
+                else {
+                    metrics0().addPutTimeNanos(duration);
+                }
+            }
+            else if (req.operation() == DELETE) {
+                if (retVal)
+                    metrics0().addRemoveAndGetTimeNanos(duration);
+                else
+                    metrics0().addRemoveTimeNanos(duration);
+            }
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
index 936c8a3bd3c87..536d0abafb792 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java
@@ -42,6 +42,8 @@
 
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
 
 /**
  * Base for near atomic update futures.
@@ -260,11 +262,34 @@ protected boolean storeFuture() {
      * @param req Request.
      */
     protected void mapSingle(UUID nodeId, GridNearAtomicAbstractUpdateRequest req) {
+        final boolean statsEnabled = cctx.config().isStatisticsEnabled();
+
+        final long start = (statsEnabled)? System.nanoTime() : 0L;
+
         if (cctx.localNodeId().equals(nodeId)) {
             cache.updateAllAsyncInternal(nodeId, req,
                 new CI2<GridNearAtomicAbstractUpdateRequest, GridNearAtomicUpdateResponse>() {
                     @Override public void apply(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicUpdateResponse res) {
                         onResult(res.nodeId(), res, false);
+
+                        if (statsEnabled && res.error() == null) {
+                            boolean retVal = req.returnValue();
+
+                            long duration = System.nanoTime() - start;
+
+                            if (req.operation() == UPDATE) {
+                                if (retVal)
+                                    cache.metrics0().addPutAndGetTimeNanos(duration);
+                                else
+                                    cache.metrics0().addPutTimeNanos(duration);
+                            }
+                            else if (req.operation() == DELETE) {
+                                if (retVal)
+                                    cache.metrics0().addRemoveAndGetTimeNanos(duration);
+                                else
+                                    cache.metrics0().addRemoveTimeNanos(duration);
+                            }
+                        }
                     }
                 });
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index 9bdd1becb7600..798b3889a8d6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -61,7 +61,9 @@
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
 
 /**
  * DHT atomic cache near update future.
@@ -588,10 +590,33 @@ private void doUpdate(Map<UUID, GridNearAtomicFullUpdateRequest> mappings) {
         }
 
         if (locUpdate != null) {
+            final boolean statsEnabled = cctx.config().isStatisticsEnabled();
+
+            final long start = (statsEnabled)? System.nanoTime() : 0L;
+
             cache.updateAllAsyncInternal(cctx.localNodeId(), locUpdate,
                 new CI2<GridNearAtomicFullUpdateRequest, GridNearAtomicUpdateResponse>() {
                     @Override public void apply(GridNearAtomicFullUpdateRequest req, GridNearAtomicUpdateResponse res) {
                         onResult(res.nodeId(), res, false);
+
+                        if (statsEnabled && res.error() == null) {
+                            boolean retVal = req.returnValue();
+
+                            long duration = System.nanoTime() - start;
+
+                            if (req.operation() == UPDATE) {
+                                if (retVal)
+                                    cache.metrics0().addPutAndGetTimeNanos(duration);
+                                else
+                                    cache.metrics0().addPutTimeNanos(duration);
+                            }
+                            else if (req.operation() == DELETE) {
+                                if (retVal)
+                                    cache.metrics0().addRemoveAndGetTimeNanos(duration);
+                                else
+                                    cache.metrics0().addRemoveTimeNanos(duration);
+                            }
+                        }
                     }
                 });
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index e1e0ec2bf087e..fc36f6102316c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -72,7 +72,6 @@
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.CX1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -289,12 +288,16 @@ public GridDistributedCacheEntry entryExx(
         if (keyCheck)
             validateCacheKeys(keys);
 
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         IgniteTxLocalAdapter tx = ctx.tm().threadLocalTx(ctx);
 
         final CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         if (tx != null && !tx.implicit() && !skipTx) {
-            return asyncOp(tx, new AsyncOp<Map<K, V>>(keys) {
+            IgniteInternalFuture<Map<K, V>> fut = asyncOp(tx, new AsyncOp<Map<K, V>>(keys) {
                 @Override public IgniteInternalFuture<Map<K, V>> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
                     return tx.getAllAsync(ctx,
                         readyTopVer,
@@ -306,6 +309,11 @@ public GridDistributedCacheEntry entryExx(
                         needVer);
                 }
             }, opCtx);
+
+            if (statsEnabled)
+                fut.listen(new UpdateGetTimeStatClosure<Map<K, V>>(metrics0(), start));
+
+            return fut;
         }
 
         AffinityTopologyVersion topVer = tx == null ?
@@ -314,7 +322,7 @@ public GridDistributedCacheEntry entryExx(
 
         subjId = ctx.subjectIdPerCall(subjId, opCtx);
 
-        return loadAsync(
+        IgniteInternalFuture<Map<K, V>> fut = loadAsync(
             ctx.cacheKeysView(keys),
             opCtx == null || !opCtx.skipStore(),
             forcePrimary,
@@ -326,6 +334,11 @@ public GridDistributedCacheEntry entryExx(
             skipVals,
             canRemap,
             needVer);
+
+        if (statsEnabled)
+            fut.listen(new UpdateGetTimeStatClosure<Map<K, V>>(metrics0(), start));
+
+        return fut;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index fb2843c02eec9..8e1c8ef79964f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -81,6 +81,9 @@ public final class GridNearGetFuture<K, V> extends CacheDistributedGetFutureAdap
     /** */
     private GridCacheVersion ver;
 
+    /** Future start time in nanoseconds. */
+    private final long startTimeNanos;
+
     /**
      * @param cctx Context.
      * @param keys Keys.
@@ -133,6 +136,8 @@ public GridNearGetFuture(
 
         ver = tx == null ? cctx.versions().next() : tx.xidVersion();
 
+        startTimeNanos = cctx.config().isStatisticsEnabled() ? System.nanoTime() : 0L;
+
         if (log == null)
             log = U.logger(cctx.kernalContext(), logRef, GridNearGetFuture.class);
     }
@@ -224,6 +229,9 @@ public void onResult(UUID nodeId, GridNearGetResponse res) {
 
             cache().dht().sendTtlUpdateRequest(expiryPlc);
 
+            if (cctx.config().isStatisticsEnabled() && !trackable && !skipVals && err == null)
+                cctx.cache().metrics0().addGetTimeNanos(System.nanoTime() - startTimeNanos);
+
             return true;
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
index 5b44d759f9080..a7ada9db19228 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
@@ -19,12 +19,15 @@
 
 import java.io.Externalizable;
 import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
 import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheOperationContext;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
@@ -234,4 +237,43 @@ void onFutureDone(GridLocalLockFuture fut) {
                 log().debug("Explicitly removed future from map of futures: " + fut);
         }
     }
+
+    /** {@inheritDoc} */
+    protected IgniteInternalFuture<Map<K, V>> getAllAsync(
+        @Nullable Collection<? extends K> keys,
+        boolean forcePrimary,
+        boolean skipTx,
+        @Nullable UUID subjId,
+        String taskName,
+        boolean deserializeBinary,
+        boolean skipVals,
+        boolean canRemap,
+        final boolean needVer
+    ) {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
+        CacheOperationContext opCtx = ctx.operationContextPerCall();
+
+        subjId = ctx.subjectIdPerCall(subjId, opCtx);
+
+        IgniteInternalFuture<Map<K, V>> fut = getAllAsync(keys,
+            null,
+            opCtx == null || !opCtx.skipStore(),
+            !skipTx,
+            subjId,
+            taskName,
+            deserializeBinary,
+            forcePrimary,
+            skipVals ? null : expiryPolicy(opCtx != null ? opCtx.expiry() : null),
+            skipVals,
+            canRemap,
+            needVer);
+
+        if (statsEnabled)
+            fut.listen(new UpdateGetTimeStatClosure<Map<K, V>>(metrics0(), start));
+
+        return fut;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index f86df2f29d684..dedd84ecef957 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -142,6 +142,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
     /** {@inheritDoc} */
     @Override protected boolean put0(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         Boolean res = (Boolean)updateAllInternal(UPDATE,
@@ -158,29 +162,50 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
         assert res != null;
 
+        if (statsEnabled && Boolean.TRUE.equals(res))
+            metrics0().addPutTimeNanos(System.nanoTime() - start);
+
         return res;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<V> getAndPutAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        return updateAllAsync0(F0.asMap(key, val),
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<V> fut = updateAllAsync0(F0.asMap(key, val),
             null,
             null,
             true,
             false,
             filter);
+
+        if (statsEnabled)
+            fut.listen(new UpdatePutAndGetTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
-        return updateAllAsync0(F0.asMap(key, val),
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<Boolean> fut = updateAllAsync0(F0.asMap(key, val),
             null,
             null,
             false,
             false,
             filter);
+
+        if (statsEnabled)
+            fut.listen(new UpdatePutTimeStatClosure<Boolean>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -230,7 +255,16 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public IgniteInternalFuture<V> getAndRemoveAsync0(K key) {
-        return removeAllAsync0(Collections.singletonList(key), true, false, null);
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
+        IgniteInternalFuture<V> fut = removeAllAsync0(Collections.singletonList(key), true, false, null);
+
+        if (statsEnabled)
+            fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
+
+        return fut;
     }
 
     /** {@inheritDoc} */
@@ -259,6 +293,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public boolean remove0(K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         Boolean rmv = (Boolean)updateAllInternal(DELETE,
@@ -275,6 +313,9 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
         assert rmv != null;
 
+        if (statsEnabled && Boolean.TRUE.equals(rmv))
+            metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+
         return rmv;
     }
 
@@ -303,6 +344,10 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
         boolean deserializeBinary,
         boolean needVer) throws IgniteCheckedException
     {
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         Map<K, V> m = getAllInternal(Collections.singleton(key),
             ctx.isSwapOrOffheapEnabled(),
             ctx.readThrough(),
@@ -313,6 +358,9 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
 
         assert m.isEmpty() || m.size() == 1 : m.size();
 
+        if (statsEnabled)
+            metrics0().addGetTimeNanos(System.nanoTime() - start);
+
         return F.firstValue(m);
     }
 
@@ -322,15 +370,24 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
         throws IgniteCheckedException {
         A.notNull(keys, "keys");
 
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         String taskName = ctx.kernalContext().job().currentTaskName();
 
-        return getAllInternal(keys,
+        Map<K, V> result = getAllInternal(keys,
             ctx.isSwapOrOffheapEnabled(),
             ctx.readThrough(),
             taskName,
             deserializeBinary,
             false,
             needVer);
+
+        if (statsEnabled)
+            metrics0().addGetTimeNanos(System.nanoTime() - start);
+
+        return result;
     }
 
     /** {@inheritDoc} */
@@ -348,15 +405,24 @@ public GridLocalAtomicCache(GridCacheContext<K, V> ctx) {
     ) {
         A.notNull(keys, "keys");
 
+        boolean statsEnabled = ctx.config().isStatisticsEnabled();
+
+        long start = statsEnabled ? System.nanoTime() : 0L;
+
         final boolean swapOrOffheap = ctx.isSwapOrOffheapEnabled();
         final boolean storeEnabled = ctx.readThrough();
 
-        return asyncOp(new Callable<Map<K, V>>() {
+        IgniteInternalFuture<Map<K, V>> fut = asyncOp(new Callable<Map<K, V>>() {
             @Override public Map<K, V> call() throws Exception {
                 return getAllInternal(keys, swapOrOffheap, storeEnabled, taskName, deserializeBinary, skipVals, needVer);
             }
         });
-    }
+
+        if (statsEnabled)
+            fut.listen(new UpdateGetTimeStatClosure<Map<K, V>>(metrics0(), start));
+
+        return fut;
+   }
 
     /**
      * Entry point to all public API get methods.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicClientServerMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicClientServerMetricsSelfTest.java
new file mode 100644
index 0000000000000..42179e194a399
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicClientServerMetricsSelfTest.java
@@ -0,0 +1,452 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.atomic;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
+
+public class GridCacheAtomicClientServerMetricsSelfTest extends GridCommonAbstractTest {
+    /** Number of nodes. */
+    private final static int GRID_CNT = 3;
+
+    /** Server node index. */
+    private static final int SERVER_NODE = 0;
+
+    /** Client node index. */
+    private static final int CLIENT_NODE = 2;
+
+    /** */
+    private static TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    private int gridCount() {
+        return GRID_CNT;
+    }
+
+    /** {@inheritDoc} */
+    protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrids(GRID_CNT - 1);
+
+        Ignition.setClientMode(true);
+
+        startGrid(CLIENT_NODE);
+
+        Ignition.setClientMode(false);
+    }
+
+    /** {@inheritDoc} */
+    protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     * Creates a new Cache Configuration for the given cache mode.
+     *
+     * @param mode Cache mode.
+     * @return Cache configuration.
+     */
+    private CacheConfiguration<Integer, Integer> getCacheConfiguration(CacheMode mode) {
+        CacheConfiguration<Integer, Integer> cacheCfg = new CacheConfiguration<>();
+
+        cacheCfg.setCacheMode(mode);
+        cacheCfg.setAtomicityMode(ATOMIC);
+        cacheCfg.setStatisticsEnabled(true);
+        cacheCfg.setName("metrics");
+
+        return cacheCfg;
+    }
+
+    /**
+     * Wait for {@link EventType#EVT_NODE_METRICS_UPDATED} event will be receieved.
+     */
+    private void awaitMetricsUpdate() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(GRID_CNT * 2);
+
+        IgnitePredicate<Event> lsnr = new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event event) {
+                latch.countDown();
+                return true;
+            }
+        };
+
+        for (int i = 0; i < gridCount(); ++i)
+            grid(i).events().localListen(lsnr, EVT_NODE_METRICS_UPDATED);
+
+        latch.await();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedGetAvgTime() throws Exception {
+        testGetAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedGetAvgTime() throws Exception {
+        testGetAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testGetAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i)
+                cache.put(i, rand.nextInt(12_000_000));
+
+            for (int i = 0; i < numOfKeys; ++i)
+                cache.get(i);
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+            assertEquals(clientMetrics.getAverageGetTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+            assertTrue(serverMetrics.getAverageGetTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedGetAndPutAvgTime() throws Exception {
+        testGetAndPutAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedGetAndPutAvgTime() throws Exception {
+        testGetAndPutAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testGetAndPutAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i)
+                cache.getAndPut(i, rand.nextInt(12_000_000));
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+            assertEquals(clientMetrics.getAverageGetTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+            assertTrue(serverMetrics.getAverageGetTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedRemoveAvgTime() throws Exception {
+        testRemoveAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedRemoveAvgTime() throws Exception {
+        testRemoveAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testRemoveAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i)
+                cache.put(i, rand.nextInt(12_000_000));
+
+            for (int i = 0; i < numOfKeys; ++i)
+                cache.remove(i);
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+            assertEquals(clientMetrics.getAverageRemoveTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+            assertTrue(serverMetrics.getAverageRemoveTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedGetAllAvgTime() throws Exception {
+        testGetAllAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedGetAllAvgTime() throws Exception {
+        testGetAllAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testGetAllAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            Set<Integer> keys = new TreeSet<>();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i) {
+                cache.put(i, rand.nextInt(12_000_000));
+
+                keys.add(i);
+            }
+
+            cache.getAll(keys);
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+            assertEquals(clientMetrics.getAverageGetTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+            assertTrue(serverMetrics.getAverageGetTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedPutAllAvgTime() throws Exception {
+        testPutAllAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedPutAllAvgTime() throws Exception {
+        testPutAllAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testPutAllAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            Map<Integer, Integer> values = new TreeMap<>();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i)
+                values .put(i, rand.nextInt(12_000_000));
+
+            cache.putAll(values);
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitionedRemoveAllAvgTime() throws Exception {
+        testRemoveAllAvgTime(PARTITIONED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicatedRemoveAllAvgTime() throws Exception {
+        testRemoveAllAvgTime(REPLICATED);
+    }
+
+    /**
+     * @param mode Cache mode.
+     * @throws Exception If failed.
+     */
+    private void testRemoveAllAvgTime(CacheMode mode) throws Exception {
+        IgniteCache<Integer, Integer> cache = null;
+
+        try {
+            cache = grid(CLIENT_NODE).getOrCreateCache(getCacheConfiguration(mode));
+
+            ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+            Set<Integer> keys = new TreeSet<>();
+
+            final int numOfKeys = 500;
+            for (int i = 0; i < numOfKeys; ++i) {
+                cache.put(i, rand.nextInt(12_000_000));
+
+                keys.add(i);
+            }
+
+            cache.removeAll(keys);
+
+            awaitMetricsUpdate();
+
+            ClusterGroup clientGroup = grid(CLIENT_NODE).cluster().forClients();
+            ClusterGroup serverGroup = grid(SERVER_NODE).cluster().forServers();
+
+            CacheMetrics clientMetrics = cache.metrics(clientGroup);
+            CacheMetrics serverMetrics = cache.metrics(serverGroup);
+
+            assertEquals(clientMetrics.getAveragePutTime(), 0.0, 0.0);
+            assertEquals(clientMetrics.getAverageRemoveTime(), 0.0, 0.0);
+
+            assertTrue(serverMetrics.getAveragePutTime() > 0.0);
+            assertTrue(serverMetrics.getAverageRemoveTime() > 0.0);
+        }
+        finally {
+            if (cache != null)
+                cache.destroy();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
index f3dd1c83d5924..5785237a103a2 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.CacheMetricsForClusterGroupSelfTest;
 import org.apache.ignite.internal.processors.cache.OffheapCacheMetricsForClusterGroupSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicClientServerMetricsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedMetricsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedTckMetricsSelfTestImpl;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearMetricsSelfTest;
@@ -58,6 +59,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheAtomicPartitionedMetricsSelfTest.class);
         suite.addTestSuite(GridCacheAtomicPartitionedTckMetricsSelfTestImpl.class);
         suite.addTestSuite(GridCacheAtomicLocalTckMetricsSelfTestImpl.class);
+        suite.addTestSuite(GridCacheAtomicClientServerMetricsSelfTest.class);
 
         // Cluster wide metrics.
         suite.addTestSuite(CacheMetricsForClusterGroupSelfTest.class);
@@ -65,4 +67,4 @@ public static TestSuite suite() throws Exception {
 
         return suite;
     }
-}
\ No newline at end of file
+}

From 0a65037ee01e6f1a552c3771fd3940b6b23827bc Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 26 Sep 2017 17:38:21 +0300
Subject: [PATCH 385/516] Backport for IGNITE-2779: BinaryMarshaller caches
 must be cleaned during client reconnect.

---
 .../ignite/internal/binary/BinaryContext.java |  15 ++
 .../CacheObjectBinaryProcessorImpl.java       |   4 +-
 ...IgniteClientReconnectBinaryContexTest.java | 185 ++++++++++++++++++
 .../IgniteClientReconnectTestSuite.java       |   2 +
 4 files changed, 205 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index b0fa68c30f0ff..bbf15bbcc4ecb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.binary;
 
+import java.util.Iterator;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
@@ -1303,6 +1304,20 @@ public void unregisterBinarySchemas() {
         schemas = null;
     }
 
+    /**
+     * Unregisters the user types descriptors.
+     **/
+    public void unregisterUserTypeDescriptors() {
+        Iterator<Map.Entry<Class<?>, BinaryClassDescriptor>> it = descByCls.entrySet().iterator();
+
+        while (it.hasNext()) {
+            Map.Entry<Class<?>, BinaryClassDescriptor> e = it.next();
+
+            if (e.getValue().userType())
+                it.remove();
+        }
+    }
+
     /**
      * Returns instance of {@link OptimizedMarshaller}.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 6b691c2236e52..5939d643aac48 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -155,9 +155,11 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
     @GridToStringExclude
     private IgniteBinary binaries;
 
-    /** Listener removes all registred binary schemas after the local client reconnected. */
+    /** Listener removes all registred binary schemas and user type descriptors after the local client reconnected. */
     private final GridLocalEventListener clientDisconLsnr = new GridLocalEventListener() {
         @Override public void onEvent(Event evt) {
+            binaryContext().unregisterUserTypeDescriptors();
+
             binaryContext().unregisterBinarySchemas();
         }
     };
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java
new file mode 100644
index 0000000000000..7401799629fac
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.Callable;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+
+/**
+ *
+ */
+public class IgniteClientReconnectBinaryContexTest extends IgniteClientReconnectAbstractTest {
+    /** */
+    public static final String DEFAULT_CACHE_NAME = "myCache";
+
+    /** {@inheritDoc} */
+    @Override protected int serverCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected int clientCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setPeerClassLoadingEnabled(false);
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectCleaningUsersMetadata() throws Exception {
+        Ignite client = grid(serverCount());
+
+        assertTrue(client.cluster().localNode().isClient());
+
+        Ignite srv = grid(0);
+
+        srv.destroyCache(DEFAULT_CACHE_NAME);
+
+        CacheConfiguration<Integer, UserClass> cacheCfg = new CacheConfiguration<Integer, UserClass>()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.REPLICATED);
+
+        IgniteCache<Integer, UserClass> cache = client.createCache(cacheCfg);
+
+        Integer key = 1;
+        UserClass val = new UserClass(1);
+
+        cache.put(key, val); // For registering user types binary metadata
+
+        reconnectServersRestart(log, client, Collections.singleton(srv), new Callable<Collection<Ignite>>() {
+            /** {@inheritDoc} */
+            @Override public Collection<Ignite> call() throws Exception {
+                return Collections.singleton((Ignite)startGrid(0));
+            }
+        });
+
+        cache = client.createCache(cacheCfg);
+
+        cache.put(key, val);
+
+        assertEquals(val, cache.get(key));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectCleaningUsersMetadata2() throws Exception {
+        Ignite client = grid(serverCount());
+
+        assertTrue(client.cluster().localNode().isClient());
+
+        Ignite srv = grid(0);
+
+        srv.destroyCache(DEFAULT_CACHE_NAME);
+
+        CacheConfiguration<Integer, UserClass> cacheCfg = new CacheConfiguration<Integer, UserClass>()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.REPLICATED);
+
+        IgniteCache<Integer, UserClass> cache = client.createCache(cacheCfg);
+
+        Integer key = 1;
+        final UserClass val = new UserClass(1);
+
+        EntryProcessor<Integer, UserClass, Object> ep = new DummyEntryProccessor(val);
+
+        cache.invoke(key, ep); // For registering user types binary metadata
+
+        reconnectServersRestart(log, client, Collections.singleton(srv), new Callable<Collection<Ignite>>() {
+            /** {@inheritDoc} */
+            @Override public Collection<Ignite> call() throws Exception {
+                return Collections.singleton((Ignite)startGrid(0));
+            }
+        });
+
+        cache = client.createCache(cacheCfg);
+
+        cache.invoke(key, ep);
+
+        assertEquals(val, cache.get(key));
+    }
+
+    /** */
+    private static class UserClass {
+        /** */
+        private final int field;
+
+        /**
+         * @param field Value.
+         */
+        private UserClass(int field) {
+            this.field = field;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            UserClass val = (UserClass)o;
+
+            return field == val.field;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return field;
+        }
+    }
+
+    /** */
+    private static class DummyEntryProccessor implements EntryProcessor<Integer, UserClass, Object> {
+        /** */
+        private final UserClass val;
+
+        /** Constructor. */
+        public DummyEntryProccessor(UserClass val) {
+            this.val = val;
+        }
+
+        /** {@inheritDoc} */
+        @Override public UserClass process(MutableEntry<Integer, UserClass> entry, Object... arguments)
+            throws EntryProcessorException {
+            entry.setValue(val);
+
+            return null;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index 67d88e1e29eac..03d3fe230c296 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.IgniteClientReconnectApiExceptionTest;
 import org.apache.ignite.internal.IgniteClientReconnectAtomicsTest;
+import org.apache.ignite.internal.IgniteClientReconnectBinaryContexTest;
 import org.apache.ignite.internal.IgniteClientReconnectCacheTest;
 import org.apache.ignite.internal.IgniteClientReconnectCollectionsTest;
 import org.apache.ignite.internal.IgniteClientReconnectComputeTest;
@@ -46,6 +47,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteClientReconnectApiExceptionTest.class);
         suite.addTestSuite(IgniteClientReconnectDiscoveryStateTest.class);
         suite.addTestSuite(IgniteClientReconnectCacheTest.class);
+        suite.addTestSuite(IgniteClientReconnectBinaryContexTest.class);
         suite.addTestSuite(IgniteClientReconnectContinuousProcessorTest.class);
         suite.addTestSuite(IgniteClientReconnectComputeTest.class);
         suite.addTestSuite(IgniteClientReconnectAtomicsTest.class);

From 39ee8af17f5cbe2db83b5ce127ac6e85c9fb4bf7 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 26 Sep 2017 17:38:21 +0300
Subject: [PATCH 386/516] Backport for IGNITE-2779: BinaryMarshaller caches
 must be cleaned during client reconnect.

---
 .../ignite/internal/binary/BinaryContext.java |  15 ++
 .../CacheObjectBinaryProcessorImpl.java       |   4 +-
 ...IgniteClientReconnectBinaryContexTest.java | 185 ++++++++++++++++++
 .../IgniteClientReconnectTestSuite.java       |   2 +
 4 files changed, 205 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index b0fa68c30f0ff..bbf15bbcc4ecb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.binary;
 
+import java.util.Iterator;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
@@ -1303,6 +1304,20 @@ public void unregisterBinarySchemas() {
         schemas = null;
     }
 
+    /**
+     * Unregisters the user types descriptors.
+     **/
+    public void unregisterUserTypeDescriptors() {
+        Iterator<Map.Entry<Class<?>, BinaryClassDescriptor>> it = descByCls.entrySet().iterator();
+
+        while (it.hasNext()) {
+            Map.Entry<Class<?>, BinaryClassDescriptor> e = it.next();
+
+            if (e.getValue().userType())
+                it.remove();
+        }
+    }
+
     /**
      * Returns instance of {@link OptimizedMarshaller}.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index f6a396992adab..62fe328882a80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -155,9 +155,11 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
     @GridToStringExclude
     private IgniteBinary binaries;
 
-    /** Listener removes all registred binary schemas after the local client reconnected. */
+    /** Listener removes all registred binary schemas and user type descriptors after the local client reconnected. */
     private final GridLocalEventListener clientDisconLsnr = new GridLocalEventListener() {
         @Override public void onEvent(Event evt) {
+            binaryContext().unregisterUserTypeDescriptors();
+
             binaryContext().unregisterBinarySchemas();
         }
     };
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java
new file mode 100644
index 0000000000000..7401799629fac
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectBinaryContexTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.Callable;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+
+/**
+ *
+ */
+public class IgniteClientReconnectBinaryContexTest extends IgniteClientReconnectAbstractTest {
+    /** */
+    public static final String DEFAULT_CACHE_NAME = "myCache";
+
+    /** {@inheritDoc} */
+    @Override protected int serverCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected int clientCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setPeerClassLoadingEnabled(false);
+
+        cfg.setMarshaller(new BinaryMarshaller());
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectCleaningUsersMetadata() throws Exception {
+        Ignite client = grid(serverCount());
+
+        assertTrue(client.cluster().localNode().isClient());
+
+        Ignite srv = grid(0);
+
+        srv.destroyCache(DEFAULT_CACHE_NAME);
+
+        CacheConfiguration<Integer, UserClass> cacheCfg = new CacheConfiguration<Integer, UserClass>()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.REPLICATED);
+
+        IgniteCache<Integer, UserClass> cache = client.createCache(cacheCfg);
+
+        Integer key = 1;
+        UserClass val = new UserClass(1);
+
+        cache.put(key, val); // For registering user types binary metadata
+
+        reconnectServersRestart(log, client, Collections.singleton(srv), new Callable<Collection<Ignite>>() {
+            /** {@inheritDoc} */
+            @Override public Collection<Ignite> call() throws Exception {
+                return Collections.singleton((Ignite)startGrid(0));
+            }
+        });
+
+        cache = client.createCache(cacheCfg);
+
+        cache.put(key, val);
+
+        assertEquals(val, cache.get(key));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectCleaningUsersMetadata2() throws Exception {
+        Ignite client = grid(serverCount());
+
+        assertTrue(client.cluster().localNode().isClient());
+
+        Ignite srv = grid(0);
+
+        srv.destroyCache(DEFAULT_CACHE_NAME);
+
+        CacheConfiguration<Integer, UserClass> cacheCfg = new CacheConfiguration<Integer, UserClass>()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.REPLICATED);
+
+        IgniteCache<Integer, UserClass> cache = client.createCache(cacheCfg);
+
+        Integer key = 1;
+        final UserClass val = new UserClass(1);
+
+        EntryProcessor<Integer, UserClass, Object> ep = new DummyEntryProccessor(val);
+
+        cache.invoke(key, ep); // For registering user types binary metadata
+
+        reconnectServersRestart(log, client, Collections.singleton(srv), new Callable<Collection<Ignite>>() {
+            /** {@inheritDoc} */
+            @Override public Collection<Ignite> call() throws Exception {
+                return Collections.singleton((Ignite)startGrid(0));
+            }
+        });
+
+        cache = client.createCache(cacheCfg);
+
+        cache.invoke(key, ep);
+
+        assertEquals(val, cache.get(key));
+    }
+
+    /** */
+    private static class UserClass {
+        /** */
+        private final int field;
+
+        /**
+         * @param field Value.
+         */
+        private UserClass(int field) {
+            this.field = field;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            UserClass val = (UserClass)o;
+
+            return field == val.field;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return field;
+        }
+    }
+
+    /** */
+    private static class DummyEntryProccessor implements EntryProcessor<Integer, UserClass, Object> {
+        /** */
+        private final UserClass val;
+
+        /** Constructor. */
+        public DummyEntryProccessor(UserClass val) {
+            this.val = val;
+        }
+
+        /** {@inheritDoc} */
+        @Override public UserClass process(MutableEntry<Integer, UserClass> entry, Object... arguments)
+            throws EntryProcessorException {
+            entry.setValue(val);
+
+            return null;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index 67d88e1e29eac..03d3fe230c296 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.IgniteClientReconnectApiExceptionTest;
 import org.apache.ignite.internal.IgniteClientReconnectAtomicsTest;
+import org.apache.ignite.internal.IgniteClientReconnectBinaryContexTest;
 import org.apache.ignite.internal.IgniteClientReconnectCacheTest;
 import org.apache.ignite.internal.IgniteClientReconnectCollectionsTest;
 import org.apache.ignite.internal.IgniteClientReconnectComputeTest;
@@ -46,6 +47,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteClientReconnectApiExceptionTest.class);
         suite.addTestSuite(IgniteClientReconnectDiscoveryStateTest.class);
         suite.addTestSuite(IgniteClientReconnectCacheTest.class);
+        suite.addTestSuite(IgniteClientReconnectBinaryContexTest.class);
         suite.addTestSuite(IgniteClientReconnectContinuousProcessorTest.class);
         suite.addTestSuite(IgniteClientReconnectComputeTest.class);
         suite.addTestSuite(IgniteClientReconnectAtomicsTest.class);

From 544e73a11feb29a499c6eebde97a1b90aa6e8ece Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Thu, 28 Sep 2017 14:14:43 +0300
Subject: [PATCH 387/516] ignite-gg-12855: Fix
 IgniteCacheTopologySafeGetSelfTest#testGetTopologySafeNodeJoinPrimaryLeave

---
 .../cache/IgniteCacheTopologySafeGetSelfTest.java        | 4 ++--
 .../ignite/testframework/junits/GridAbstractTest.java    | 9 +++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTopologySafeGetSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTopologySafeGetSelfTest.java
index 674f2b0adf503..6da99bbf7c5a3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTopologySafeGetSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTopologySafeGetSelfTest.java
@@ -147,7 +147,7 @@ public void checkGetTopologySafeNodeJoin(boolean failPrimary) throws Exception {
             IgniteInternalFuture<?> nodeFut = startNodeAsync();
 
             if (failPrimary)
-                stopGrid(1);
+                stopGrid(1, false, false);
 
             assertEquals(key, ((IgniteKernal)ignite(0)).internalCache("tx").getTopologySafe(key));
             assertEquals(key, ((IgniteKernal)ignite(0)).internalCache("atomic").getTopologySafe(key));
@@ -219,4 +219,4 @@ private void releaseTx() {
 
         releaseLatch.countDown();
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index e853b5046c094..22c796f6cd3e1 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -1182,6 +1182,15 @@ protected void stopGrid(int idx, boolean cancel) {
         }
     }
 
+    /**
+     * @param idx Index of the grid to stop.
+     * @param cancel Cancel flag.
+     * @param awaitTop Await topology change flag.
+     */
+    protected void stopGrid(int idx, boolean cancel, boolean awaitTop) {
+        stopGrid(getTestGridName(idx), false, awaitTop);
+    }
+
     /**
      * @param idx Index of the grid to stop.
      */

From ac8379ef55b7f091b97707e107d967544988493c Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Thu, 28 Sep 2017 16:21:34 +0300
Subject: [PATCH 388/516] GG-12826 Backport fixes for NPE in
 GridDhtPartitionDemander.

---
 .../preloader/GridDhtPartitionDemander.java   | 43 +++++++++----------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 9d12d0edc97a2..ab2dae917d1f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -406,26 +406,35 @@ else if (delay > 0) {
     }
 
     /**
-     * @param fut Future.
+     * @param fut Rebalance future.
      * @param assigns Assignments.
-     * @throws IgniteCheckedException If failed.
-     * @return Partitions were requested.
      */
     private void requestPartitions(
         final RebalanceFuture fut,
         GridDhtPreloaderAssignments assigns){
+        assert fut != null;
+
         if (topologyChanged(fut)) {
             fut.cancel();
 
             return;
         }
 
-        for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assigns.entrySet()) {
-            final ClusterNode node = e.getKey();
+        synchronized (fut) {
+            if (fut.isDone())
+                return;
 
-            GridDhtPartitionDemandMessage d = e.getValue();
+            // Must add all remaining node before send first request, for avoid race between add remaining node and
+            // processing response, see checkIsDone(boolean).
+            for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assigns.entrySet()) {
+                UUID nodeId = e.getKey().id();
 
-            fut.appendPartitions(node.id(), d.partitions()); //Future preparation.
+                Collection<Integer> parts = e.getValue().partitions();
+
+                assert parts != null : "Partitions are null [cache=" + cctx.name() + ", fromNode=" + nodeId + "]";
+
+                fut.remaining.put(nodeId, new T2<>(U.currentTimeMillis(), parts));
+            }
         }
 
         for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assigns.entrySet()) {
@@ -433,10 +442,10 @@ private void requestPartitions(
 
             final CacheConfiguration cfg = cctx.config();
 
-            final Collection<Integer> parts = fut.remaining.get(node.id()).get2();
-
             GridDhtPartitionDemandMessage d = e.getValue();
 
+            final Collection<Integer> parts = d.partitions();
+
             //Check remote node rebalancing API version.
             if (node.version().compareTo(GridDhtPreloader.REBALANCING_VER_2_SINCE) >= 0) {
                 U.log(log, "Starting rebalancing [mode=" + cfg.getRebalanceMode() +
@@ -445,7 +454,7 @@ private void requestPartitions(
 
                 int lsnrCnt = cctx.gridConfig().getRebalanceThreadPoolSize();
 
-            final List<Set<Integer>> sParts = new ArrayList<>(lsnrCnt);
+                final List<Set<Integer>> sParts = new ArrayList<>(lsnrCnt);
 
                 for (int cnt = 0; cnt < lsnrCnt; cnt++)
                     sParts.add(new HashSet<Integer>());
@@ -472,6 +481,8 @@ private void requestPartitions(
                             @Override public void run() {
                                 try {
                                     if (!fut.isDone()) {
+                                        // Future can be already cancelled at this moment and all failovers happened.
+                                        // New requests will not be covered by failovers.
                                         cctx.io().sendOrderedMessage(node,
                                             rebalanceTopics.get(finalCnt), initD, cctx.ioPolicy(), initD.timeout());
 
@@ -915,18 +926,6 @@ private boolean isInitial() {
             return topVer == null;
         }
 
-        /**
-         * @param nodeId Node id.
-         * @param parts Parts.
-         */
-        private void appendPartitions(UUID nodeId, Collection<Integer> parts) {
-            synchronized (this) {
-                assert parts != null : "Partitions are null [cache=" + cctx.name() + ", fromNode=" + nodeId + "]";
-
-                remaining.put(nodeId, new T2<>(U.currentTimeMillis(), parts));
-            }
-        }
-
         /**
          * Cancels this future.
          *

From 1df9b02284516707133237f5fe1280d992d0fb89 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Fri, 29 Sep 2017 18:03:51 +0300
Subject: [PATCH 389/516] Merge branch 'ignite-1.8.11' into ignite-1.9.7

---
 .../internal/processors/service/GridServiceProcessor.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 2ef6aa856ab86..a22ccaad367ff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -565,7 +565,7 @@ public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs)
 
         if (!busyLock.enterBusy()) {
             IgniteCheckedException e = new IgniteCheckedException("Service deployment has been cancelled (node is stopping): " +
-                cfg.getName());
+                cfgs);
 
             return new GridFinishedFuture<>(e);
         }

From 890be7a55a0fd6af6b19dee0ca923706f8039e5a Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 3 Oct 2017 09:57:36 +0300
Subject: [PATCH 390/516] GG-12891 - Event EVT_CACHE_REBALANCE_PART_DATA_LOST
 should be fired only once

---
 .../dht/GridDhtPartitionTopologyImpl.java     |  22 ++-
 .../GridDhtPartitionsExchangeFuture.java      |   6 +-
 .../GridLostPartitionRebalanceTest.java       | 140 +++++++++++++++---
 3 files changed, 130 insertions(+), 38 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index eb6658bd2a01b..f790ac64e6cd1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -604,14 +604,6 @@ else if (!node2part.nodeId().equals(loc.id())) {
 
                                 changed = true;
 
-                                if (cctx.events().isRecordable(EVT_CACHE_REBALANCE_PART_DATA_LOST)) {
-                                    DiscoveryEvent discoEvt = exchFut.discoveryEvent();
-
-                                    cctx.events().addPreloadEvent(p,
-                                        EVT_CACHE_REBALANCE_PART_DATA_LOST, discoEvt.eventNode(),
-                                        discoEvt.type(), discoEvt.timestamp());
-                                }
-
                                 if (log.isDebugEnabled())
                                     log.debug("Owned partition: " + locPart);
                             }
@@ -1237,6 +1229,10 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
 
     /** {@inheritDoc} */
     @Override public void detectLostPartitions(DiscoveryEvent discoEvt) {
+        // Method generates events only.
+        if (!cctx.events().isRecordable(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST))
+            return;
+
         lock.writeLock().lock();
 
         try {
@@ -1255,7 +1251,9 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
 
                         GridDhtPartitionState state = partMap.get(p);
 
-                        if (state == OWNING) {
+                        // We need to extra check owners() using discovery,
+                        // because on coordinator a new owner is already determined.
+                        if (state == OWNING && !F.isEmpty(owners(p))) {
                             foundOwner = true;
 
                             break;
@@ -1272,11 +1270,9 @@ private List<ClusterNode> ownersAndMoving(int p, AffinityTopologyVersion topVer)
             }
 
             if (lost != null) {
-                // Update partition state on all nodes.
                 for (Integer part : lost) {
-                    if (cctx.events().isRecordable(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST))
-                        cctx.events().addPreloadEvent(part, EVT_CACHE_REBALANCE_PART_DATA_LOST,
-                            discoEvt.eventNode(), discoEvt.type(), discoEvt.timestamp());
+                    cctx.events().addPreloadEvent(part, EVT_CACHE_REBALANCE_PART_DATA_LOST,
+                        discoEvt.eventNode(), discoEvt.type(), discoEvt.timestamp());
                 }
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 0de932780a1ec..13fdb30137f58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1102,8 +1102,7 @@ private void sendPartitions(ClusterNode oldestNode) {
             }
 
             if (discoEvt.type() == EVT_NODE_LEFT ||
-                discoEvt.type() == EVT_NODE_FAILED ||
-                discoEvt.type() == EVT_NODE_JOINED)
+                discoEvt.type() == EVT_NODE_FAILED)
                 detectLostPartitions();
 
             Map<Integer, Boolean> m = null;
@@ -1441,9 +1440,6 @@ private void onAllReceived() {
                 }
             }
 
-            if (discoEvt.type() == EVT_NODE_LEFT || discoEvt.type() == EVT_NODE_FAILED)
-                detectLostPartitions();
-
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
                 if (!cacheCtx.isLocal())
                     cacheCtx.topology().checkEvictions();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java
index e063b1c138f48..56eed411ce56b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java
@@ -19,13 +19,18 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -33,11 +38,15 @@
 import org.apache.ignite.events.CacheRebalancingEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
 
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
 
@@ -46,14 +55,23 @@
  */
 public class GridLostPartitionRebalanceTest extends GridCommonAbstractTest {
     /** Cache name. */
-    private static final String CACHE_NAME = "cache";
+    private static final String CACHE_NAME = "test";
 
     /** Latch. */
-    private static CountDownLatch latch = new CountDownLatch(4);
+    private static CountDownLatch latch;
+
+    /** Count. */
+    private static AtomicInteger cnt;
 
     /** Failed flag. */
     private static boolean failed;
 
+    /** Backups. */
+    private int backups;
+
+    /** Expected events. */
+    private int expEvts;
+
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
@@ -63,6 +81,8 @@ public class GridLostPartitionRebalanceTest extends GridCommonAbstractTest {
 
         ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
         ccfg.setNodeFilter(NODE_FILTER);
+        ccfg.setBackups(backups);
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
 
         cfg.setCacheConfiguration(ccfg);
 
@@ -100,10 +120,36 @@ public class GridLostPartitionRebalanceTest extends GridCommonAbstractTest {
         }
     };
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        latch = new CountDownLatch(expEvts);
+        cnt = new AtomicInteger(0);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartDataLostEvent1Backup() throws Exception {
+        expEvts = 3;
+        backups = 1;
+
+        checkEvents();
+    }
+
     /**
      * @throws Exception If failed.
      */
-    public void testPartDataLostEvent() throws Exception {
+    public void testPartDataLostEventNoBackups() throws Exception {
+        expEvts = 4;
+        backups = 0;
+
+        checkEvents();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkEvents() throws Exception {
         List<Ignite> srvrs = new ArrayList<>();
 
         // Client router. It always up, so client is guaranteed to get
@@ -124,23 +170,80 @@ public void testPartDataLostEvent() throws Exception {
             cache.put(i, i);
 
         // Stop node with 0 partition.
-        ClusterNode node = client.affinity(CACHE_NAME).mapPartitionToNode(0);
+        Set<ClusterNode> nodes = new HashSet<>(client.affinity(CACHE_NAME).mapPartitionToPrimaryAndBackups(0));
 
-        for (Ignite srv : srvrs) {
-            if (node.equals(srv.cluster().localNode())) {
-                srv.close();
-
-                System.out.println(">> Stopped " + srv.name());
-
-                break;
-            }
-        }
+        final List<String> stopped = stopAffinityNodes(srvrs, nodes);
 
         // Check that all nodes (and clients) got notified.
         assert latch.await(15, TimeUnit.SECONDS) : latch.getCount();
 
         // Check that exchange was not finished when event received.
         assertFalse("Exchange was finished when event received.", failed);
+
+        U.sleep(4_000);
+
+        assertEquals("Fired more events than expected", expEvts, cnt.get());
+
+        startNodes(stopped);
+
+        assertEquals("Fired unexpected events", expEvts, cnt.get());
+    }
+
+    /**
+     * @param nodeNames Node names.
+     */
+    private void startNodes(List<String> nodeNames) throws Exception {
+        List<IgniteInternalFuture<?>> futs = new ArrayList<>();
+
+        for (final String nodeName : nodeNames) {
+            IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    startGrid(nodeName);
+
+                    return null;
+                }
+            });
+
+            futs.add(fut);
+        }
+
+        for (IgniteInternalFuture<?> fut : futs)
+            fut.get();
+    }
+
+    /**
+     * @param srvrs Servers.
+     * @param nodes Nodes.
+     */
+    @NotNull private List<String> stopAffinityNodes(List<Ignite> srvrs, Set<ClusterNode> nodes) throws IgniteCheckedException {
+        List<IgniteInternalFuture<?>> futs = new ArrayList<>();
+
+        final List<String> stopped = new ArrayList<>();
+
+        for (final Ignite srv : srvrs) {
+            final ClusterNode node = srv.cluster().localNode();
+
+            if (nodes.contains(node)) {
+                IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+                    @Override public Object call() throws Exception {
+                        srv.close();
+
+                        System.out.println(">> Stopped " + srv.name() + " " + node.id());
+
+                        stopped.add(srv.name());
+
+                        return null;
+                    }
+                });
+
+                futs.add(fut);
+            }
+        }
+
+        for (IgniteInternalFuture<?> fut : futs)
+            fut.get();
+
+        return stopped;
     }
 
     /**
@@ -155,24 +258,21 @@ private static class Listener implements IgnitePredicate<CacheRebalancingEvent>
         @IgniteInstanceResource
         private Ignite ignite;
 
-        /** Got. */
-        private final AtomicBoolean got = new AtomicBoolean(false);
-
         /** {@inheritDoc} */
         @Override public boolean apply(CacheRebalancingEvent evt) {
             int part = evt.partition();
 
             // AtomicBoolean because new owner will produce two events.
-            if (part == 0 && CACHE_NAME.equals(evt.cacheName()) && got.compareAndSet(false, true)) {
+            if (part == 0 && CACHE_NAME.equals(evt.cacheName())) {
                 System.out.println(">> Received event for 0 partition. [node=" + ignite.name() + ", evt=" + evt
                     + ", thread=" + Thread.currentThread().getName() + ']');
 
                 latch.countDown();
 
+                cnt.incrementAndGet();
+
                 if (exchangeCompleted(ignite))
                     failed = true;
-
-                return false;
             }
 
             return true;

From 955ae62887e652d819d3ba19e92921d94e6c8d7e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 3 Oct 2017 12:32:30 +0300
Subject: [PATCH 391/516] GG-12775: Backport fixes for NPE in
 GridPArtitionDemander. Unmute test.

---
 .../discovery/GridDiscoveryManagerAliveCacheSelfTest.java       | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
index f0c50ebeb3bf1..1c048b7e8466c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
@@ -169,8 +169,6 @@ public void testAlives() throws Exception {
      * @throws Exception If failed.
      */
     public void testAlivesClient() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-1583");
-
         clientMode = true;
 
         doTestAlive();

From 92153893ee5056f133337a1bb90451965a3422d4 Mon Sep 17 00:00:00 2001
From: Andrey Kuznetsov <stkuzma@gmail.com>
Date: Fri, 11 Aug 2017 15:51:31 +0300
Subject: [PATCH 392/516] IGNITE-6009: Fixed IgniteSemaphore test.

(cherry picked from commit dadc42f)
---
 .../IgniteSemaphoreExample.java               | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/examples/src/main/java/org/apache/ignite/examples/datastructures/IgniteSemaphoreExample.java b/examples/src/main/java/org/apache/ignite/examples/datastructures/IgniteSemaphoreExample.java
index 1c078b02318bd..daa1023e1b56b 100644
--- a/examples/src/main/java/org/apache/ignite/examples/datastructures/IgniteSemaphoreExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/datastructures/IgniteSemaphoreExample.java
@@ -37,12 +37,6 @@ public class IgniteSemaphoreExample {
     /** Number of items for each producer/consumer to produce/consume. */
     private static final int OPS_COUNT = 100;
 
-    /** Number of producers. */
-    private static final int NUM_PRODUCERS = 10;
-
-    /** Number of consumers. */
-    private static final int NUM_CONSUMERS = 10;
-
     /** Synchronization semaphore name. */
     private static final String SEM_NAME = IgniteSemaphoreExample.class.getSimpleName();
 
@@ -53,6 +47,12 @@ public class IgniteSemaphoreExample {
      */
     public static void main(String[] args) {
         try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+            // Number of producers; should be equal to number of consumers.
+            // This value should not exceed overall number of compute threads in a cluster,
+            // otherwise blocking consumer jobs can occupy all the threads leading to deadlock.
+            int producerConsumerCount =
+                ignite.configuration().getPublicThreadPoolSize() * ignite.cluster().forServers().nodes().size() / 2;
+
             System.out.println();
             System.out.println(">>> Cache atomic semaphore example started.");
 
@@ -66,17 +66,17 @@ public static void main(String[] args) {
             IgniteSemaphore semaphore = ignite.semaphore(semaphoreName, 0, false, true);
 
             // Start consumers on all cluster nodes.
-            for (int i = 0; i < NUM_CONSUMERS; i++)
+            for (int i = 0; i < producerConsumerCount; i++)
                 ignite.compute().withAsync().run(new Consumer(semaphoreName));
 
             // Start producers on all cluster nodes.
-            for (int i = 0; i < NUM_PRODUCERS; i++)
+            for (int i = 0; i < producerConsumerCount; i++)
                 ignite.compute().withAsync().run(new Producer(semaphoreName));
 
             System.out.println("Master node is waiting for all other nodes to finish...");
 
             // Wait for everyone to finish.
-            syncSemaphore.acquire(NUM_CONSUMERS + NUM_PRODUCERS);
+            syncSemaphore.acquire(2 * producerConsumerCount);
         }
 
         System.out.flush();
@@ -159,7 +159,7 @@ public Consumer(String semaphoreName) {
             System.out.println("Consumer finished [nodeId=" + Ignition.ignite().cluster().localNode().id() + ']');
 
             // Gets the syncing semaphore
-            IgniteSemaphore sync = Ignition.ignite().semaphore(SEM_NAME, 3, true, true);
+            IgniteSemaphore sync = Ignition.ignite().semaphore(SEM_NAME, 0, true, true);
 
             // Signals the master thread.
             sync.release();

From 8c17b0efa664f8552b64d16dae2a3a03e2732347 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 5 Oct 2017 15:18:21 +0300
Subject: [PATCH 393/516] Backport of IGNITE-6545: Failure during Ignite
 Service.cancel() can break normal shutdown process.

---
 .../processors/service/GridServiceProcessor.java      | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index a22ccaad367ff..4ebe1b00d042d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -337,7 +337,16 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
             Service svc = ctx.service();
 
             if (svc != null)
-                svc.cancel(ctx);
+                try {
+                    svc.cancel(ctx);
+                }
+                catch (Throwable e) {
+                    log.error("Failed to cancel service (ignoring) [name=" + ctx.name() +
+                        ", execId=" + ctx.executionId() + ']', e);
+
+                    if (e instanceof Error)
+                        throw e;
+                }
 
             ctx.executor().shutdownNow();
         }

From 74aa02b8faa2c871da40ff2ad0e93afc019702b9 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 6 Oct 2017 19:09:50 +0300
Subject: [PATCH 394/516] Backport of IGNITE-6562: Dynamic service deployment
 should use projection if NodeFilter is not set.

---
 .../ignite/internal/IgniteServicesImpl.java   |  2 +-
 .../service/GridServiceProcessor.java         | 60 +++++++++-------
 .../GridServiceProcessorAbstractSelfTest.java |  5 +-
 ...GridServiceProcessorMultiNodeSelfTest.java | 71 ++++++++++++++++++-
 4 files changed, 106 insertions(+), 32 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
index 58b3a2a96ff1c..0141aea02c918 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
@@ -161,7 +161,7 @@ public IgniteServicesImpl(GridKernalContext ctx, ClusterGroupAdapter prj, boolea
         guard();
 
         try {
-            saveOrGet(ctx.service().deployAll(cfgs));
+            saveOrGet(ctx.service().deployAll(prj, cfgs));
         }
         catch (IgniteCheckedException e) {
             throw U.convertException(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 4ebe1b00d042d..5e6a0032cdbec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -90,6 +90,7 @@
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
@@ -298,15 +299,9 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
 
         ServiceConfiguration[] cfgs = ctx.config().getServiceConfiguration();
 
-        if (cfgs != null) {
-            for (ServiceConfiguration c : cfgs) {
-                // Deploy only on server nodes by default.
-                if (c.getNodeFilter() == null)
-                    c.setNodeFilter(ctx.cluster().get().forServers().predicate());
-            }
-
-            deployAll(Arrays.asList(cfgs)).get();
-        }
+        if (cfgs != null)
+            // Deploy only on server nodes by default.
+            deployAll(Arrays.asList(cfgs), ctx.cluster().get().forServers().predicate()).get();
 
         if (log.isDebugEnabled())
             log.debug("Started service processor.");
@@ -467,9 +462,8 @@ public IgniteInternalFuture<?> deployMultiple(ClusterGroup prj, String name, Ser
         cfg.setService(svc);
         cfg.setTotalCount(totalCnt);
         cfg.setMaxPerNodeCount(maxPerNodeCnt);
-        cfg.setNodeFilter(F.<ClusterNode>alwaysTrue() == prj.predicate() ? null : prj.predicate());
 
-        return deploy(cfg);
+        return deployAll(prj, Collections.singleton(cfg));
     }
 
     /**
@@ -491,14 +485,17 @@ public IgniteInternalFuture<?> deployKeyAffinitySingleton(String name, Service s
         cfg.setTotalCount(1);
         cfg.setMaxPerNodeCount(1);
 
-        return deploy(cfg);
+        // Ignore projection here.
+        return deployAll(Collections.singleton(cfg), null);
     }
 
     /**
      * @param cfgs Service configurations.
+     * @param dfltNodeFilter Default NodeFilter.
      * @return Configurations to deploy.
      */
-    private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceConfiguration> cfgs) {
+    private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceConfiguration> cfgs,
+        IgnitePredicate<ClusterNode> dfltNodeFilter) {
         List<ServiceConfiguration> cfgsCp = new ArrayList<>(cfgs.size());
 
         ServicesCompatibilityState state = markCompatibilityStateAsUsed();
@@ -510,6 +507,11 @@ private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceCo
         for (ServiceConfiguration cfg : cfgs) {
             Exception err = null;
 
+            // Deploy to projection node by default
+            // or only on server nodes if no projection .
+            if (cfg.getNodeFilter() == null && dfltNodeFilter != null)
+                cfg.setNodeFilter(dfltNodeFilter);
+
             try {
                 validate(cfg);
             }
@@ -565,11 +567,28 @@ private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceCo
         return new PreparedConfigurations(cfgsCp, failedFuts);
     }
 
+    /**
+     * @param prj Grid projection.
+     * @param cfgs Service configurations.
+     * @return Future for deployment.
+     */
+    public IgniteInternalFuture<?> deployAll(ClusterGroup prj, Collection<ServiceConfiguration> cfgs) {
+        if (prj == null)
+            // Deploy to servers by default if no projection specified.
+            return deployAll(cfgs,  ctx.cluster().get().forServers().predicate());
+        else if (prj.predicate() == F.<ClusterNode>alwaysTrue())
+            return deployAll(cfgs,  null);
+        else
+            // Deploy to predicate nodes by default.
+            return deployAll(cfgs,  prj.predicate());
+    }
+
     /**
      * @param cfgs Service configurations.
+     * @param dfltNodeFilter Default NodeFilter.
      * @return Future for deployment.
      */
-    public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs) {
+    private IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs, IgnitePredicate<ClusterNode> dfltNodeFilter) {
         assert cfgs != null;
 
         if (!busyLock.enterBusy()) {
@@ -580,8 +599,7 @@ public IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs)
         }
 
         try {
-
-            PreparedConfigurations srvCfg = prepareServiceConfigurations(cfgs);
+            PreparedConfigurations srvCfg = prepareServiceConfigurations(cfgs, dfltNodeFilter);
 
             List<ServiceConfiguration> cfgsCp = srvCfg.cfgs;
 
@@ -745,16 +763,6 @@ private void writeServiceToCache(GridServiceDeploymentCompoundFuture res, Servic
         }
     }
 
-    /**
-     * @param cfg Service configuration.
-     * @return Future for deployment.
-     */
-    public IgniteInternalFuture<?> deploy(ServiceConfiguration cfg) {
-        A.notNull(cfg, "cfg");
-
-        return deployAll(Collections.singleton(cfg));
-    }
-
     /**
      * @return Compatibility state.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
index 6d91f36aab00b..debe4859d1c19 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
@@ -107,10 +107,7 @@ protected ServiceConfiguration[] services() {
     /** {@inheritDoc} */
     @SuppressWarnings("ConstantConditions")
     @Override protected void beforeTestsStarted() throws Exception {
-        assert nodeCount() >= 1;
-
-        for (int i = 0; i < nodeCount(); i++)
-            startGrid(i);
+        startGridsMultiThreaded(nodeCount());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
index 2958b7cb248c8..57b584f672f56 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorMultiNodeSelfTest.java
@@ -175,7 +175,7 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
                 // Since we start extra nodes, there may be extra start and cancel events,
                 // so we check only the difference between start and cancel and
                 // not start and cancel events individually.
-                assertEquals(name, nodeCount() + servers,  DummyService.started(name) - DummyService.cancelled(name));
+                assertEquals(name, nodeCount() + servers, DummyService.started(name) - DummyService.cancelled(name));
 
                 checkCount(name, g.services().serviceDescriptors(), nodeCount() + servers);
             }
@@ -188,6 +188,75 @@ public void testDeployOnEachNodeButClientUpdateTopology() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOnEachProjectionNodeUpdateTopology() throws Exception {
+        // Prestart client node.
+        Ignite client = startGrid("client", getConfiguration("client").setClientMode(true));
+
+        try {
+            final String name = "serviceOnEachProjectionNodeUpdateTopology";
+
+            Ignite g = randomGrid();
+
+            int prestartedSrvcs = g.cluster().forClients().nodes().size();
+
+            CountDownLatch latch = new CountDownLatch(prestartedSrvcs);
+
+            DummyService.exeLatch(name, latch);
+
+            IgniteServices svcs = g.services(g.cluster().forClients()).withAsync();
+
+            svcs.deployNodeSingleton(name, new DummyService());
+
+            IgniteFuture<?> fut = svcs.future();
+
+            info("Deployed service: " + name);
+
+            fut.get();
+
+            info("Finished waiting for service future: " + name);
+
+            latch.await();
+
+            // Ensure service is deployed
+            assertNotNull(client.services().serviceProxy(name, Service.class, false, 2000));
+
+            assertEquals(name, prestartedSrvcs, DummyService.started(name));
+            assertEquals(name, 0, DummyService.cancelled(name));
+
+            int servers = 2;
+
+            int clients = 2;
+
+            latch = new CountDownLatch(clients);
+
+            DummyService.exeLatch(name, latch);
+
+            startExtraNodes(servers, clients);
+
+            try {
+                latch.await();
+
+                waitForDeployment(name, clients);
+
+                // Since we start extra nodes, there may be extra start and cancel events,
+                // so we check only the difference between start and cancel and
+                // not start and cancel events individually.
+                assertEquals(name, clients + prestartedSrvcs, DummyService.started(name) - DummyService.cancelled(name));
+
+                checkCount(name, g.services().serviceDescriptors(), clients + prestartedSrvcs);
+            }
+            finally {
+                stopExtraNodes(servers + clients);
+            }
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From dd14cf4ac50e11ecaec1ae0e97a6ccd3bec9ef77 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Tue, 21 Mar 2017 14:15:52 +0300
Subject: [PATCH 395/516] More info in toString for lock futures.

(cherry picked from commit 4322e00)
---
 .../distributed/dht/colocated/GridDhtColocatedLockFuture.java    | 1 +
 .../processors/cache/distributed/near/GridNearLockFuture.java    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index 79ca1087a3b88..4c37c538d4950 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -105,6 +105,7 @@ public final class GridDhtColocatedLockFuture extends GridCompoundIdentityFuture
     private final long threadId;
 
     /** Keys to lock. */
+    @GridToStringInclude
     private Collection<KeyCacheObject> keys;
 
     /** Future ID. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index d3e3a15831bb2..d1f3fac1b5416 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -100,6 +100,7 @@ public final class GridNearLockFuture extends GridCompoundIdentityFuture<Boolean
     private long threadId;
 
     /** Keys to lock. */
+    @GridToStringInclude
     private final Collection<KeyCacheObject> keys;
 
     /** Future ID. */

From d3dee460c4c94d48e65d285ea1ec045f507a5d8c Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 11 Oct 2017 14:27:21 +0300
Subject: [PATCH 396/516] GG-12920 - Fix: After ExchangeActions backport:
 CacheGetInsideLockChangingTopologyTest is broken

---
 .../ignite/internal/processors/cache/ExchangeActions.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
index 0f5f914a2c003..c64efe9fb3976 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
@@ -83,7 +83,7 @@ public boolean cacheStopped(int cacheId) {
     public boolean cacheStarted(int cacheId) {
         if (cachesToStart != null) {
             for (CacheActionData cache : cachesToStart.values()) {
-                if (cache.desc.cacheId() == cacheId)
+                if (cache.desc.cacheId() == cacheId && !cache.request().clientStartOnly())
                     return true;
             }
         }

From 67b19157b4bb95e66b0009e71a572c2577fb968a Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 6 Oct 2017 15:26:21 +0300
Subject: [PATCH 397/516] GG-12398: Backport test fix
 GridCacheAtomicInvalidPartitionHandlingSelfTest.testClockFullAsync.

(cherry picked from commit 0519fb7)

(cherry picked from commit fd88aef)
---
 .../GridCacheAtomicInvalidPartitionHandlingSelfTest.java       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java
index 3fd4dd8e586eb..ff98bc1550a9a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java
@@ -354,7 +354,8 @@ private void checkRestarts(CacheAtomicWriteOrderMode writeOrder,
             for (int r = 0; r < 20; r++) {
                 int idx0 = rnd.nextInt(gridCnt - 1) + 1;
 
-                stopGrid(idx0);
+                // No wait for PartitionMapExchange
+                stopGrid(idx0, false, false);
 
                 U.sleep(200);
 

From d6232fb1fe079cdb92ef261da3b7aa591a144078 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 12 Oct 2017 20:32:11 +0300
Subject: [PATCH 398/516] GG-12703: Missed tests added.

---
 ...iteClientReconnectContinuousProcessorTest.java | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
index c86dcabb3a208..0ff5883d11b57 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
@@ -23,7 +23,6 @@
 import javax.cache.event.CacheEntryUpdatedListener;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.query.ContinuousQuery;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -111,6 +110,20 @@ public void testEventListenerReconnect() throws Exception {
         assertFalse(lsnr.latch.await(3000, MILLISECONDS));
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMessageListenerReconnectAndStopFromServer() throws Exception {
+        testMessageListenerReconnect(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMessageListenerReconnectAndStopFromClient() throws Exception {
+        testMessageListenerReconnect(true);
+    }
+
     /**
      * @param stopFromClient If {@code true} stops listener from client node, otherwise from server.
      * @throws Exception If failed.

From 3cc9435fc2b4e25e35dcbb96774868472bc41dbf Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 13 Oct 2017 12:53:13 +0300
Subject: [PATCH 399/516] GG-12942: Fixed Fix Cache Failover 2 suite tests.

Fixed tests:
GridCachePartitionedTxSalvageSelfTest.testOptimisticTxSalvageBeforeTimeout
GridCachePartitionedTxSalvageSelfTest.testPessimisticcTxSalvageBeforeTimeout
---
 .../distributed/near/GridCachePartitionedTxSalvageSelfTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java
index c2de8c576c597..5a093440ceeb3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java
@@ -235,7 +235,7 @@ private void startTxAndPutKeys(final TransactionConcurrency mode, final boolean
      * @throws Exception If failed.
      */
     private void stopNodeAndSleep(long timeout) throws Exception {
-        stopGrid(0);
+        stopGrid(0, false, false);
 
         info("Stopped grid.");
 

From e46a9d1276bd98dc1d3c5f4d45567e1ffd3b169e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 13 Oct 2017 13:36:06 +0300
Subject: [PATCH 400/516] GG-12941: Fixed Ignite Cache 5 suite hanging.

Fixed GridCachePartitionEvictionDuringReadThroughSelfTest.

(cherry picked from commit 3cc9435)
---
 ...tionEvictionDuringReadThroughSelfTest.java | 68 +++++++++++++------
 1 file changed, 47 insertions(+), 21 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java
index d5351f7e37776..5fef40024e0e4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java
@@ -18,8 +18,8 @@
 package org.apache.ignite.internal.processors.cache.distributed;
 
 import java.util.LinkedHashSet;
+import java.util.Set;
 import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.cache.Cache;
@@ -36,6 +36,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -43,6 +44,12 @@
  *
  */
 public class GridCachePartitionEvictionDuringReadThroughSelfTest extends GridCommonAbstractTest {
+    /** Failing key. */
+    private static final int FAILING_KEY = 3;
+
+    /** Data read grid index. */
+    private static final int DATA_READ_GRID_IDX = 0;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -75,28 +82,37 @@ public class GridCachePartitionEvictionDuringReadThroughSelfTest extends GridCom
      * @throws Exception if failed.
      */
     public void testPartitionRent() throws Exception {
-        startGrid(0);
+//        fail("https://issues.apache.org/jira/browse/IGNITE-5759");
+
+        startGrid(DATA_READ_GRID_IDX);
 
         final AtomicBoolean done = new AtomicBoolean();
 
-        IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Integer>() {
+        IgniteInternalFuture<Long> gridAndCacheAccessFut = GridTestUtils.runMultiThreadedAsync(new Callable<Integer>() {
             @Override
             public Integer call() throws Exception {
-                LinkedHashSet<Integer> set = new LinkedHashSet<>();
+                final Set<Integer> keysSet = new LinkedHashSet<>();
 
-                set.add(1);
-                set.add(2);
-                set.add(3);
-                set.add(4);
-                set.add(5);
+                keysSet.add(1);
+                keysSet.add(2);
+                keysSet.add(FAILING_KEY);
+                keysSet.add(4);
+                keysSet.add(5);
 
                 while (!done.get()) {
                     try {
-                        grid(0).<Integer, Integer>cache("config").getAll(set);
+                        grid(DATA_READ_GRID_IDX).<Integer, Integer>cache("config").getAll(keysSet);
                     }
-                    catch (Throwable ignore) {
-                        // No-op.
+                    catch (Throwable ex) {
+                        if (ex instanceof Error)
+                            throw ex;
+
+                        if (ex instanceof InterruptedException)
+                            Thread.currentThread().interrupt();
                     }
+
+                    if (Thread.currentThread().isInterrupted())
+                        throw new IgniteInterruptedCheckedException("Execution of [" + Thread.currentThread().getName() + "] Interrupted. Test is probably timed out");
                 }
 
                 return null;
@@ -115,11 +131,14 @@ public Integer call() throws Exception {
             }
         });
 
-        startFut.get();
-
-        done.set(true);
+        try {
+            startFut.get();
+        }
+        finally {
+            done.set(true);
+        }
 
-        fut.get();
+        gridAndCacheAccessFut.get();
     }
 
     /**
@@ -136,13 +155,10 @@ private static class CacheStoreFactory implements Factory<CacheStore<Integer, In
      *
      */
     private static class HangingCacheStore extends CacheStoreAdapter<Integer, Integer> {
-        /** */
-        private CountDownLatch releaseLatch = new CountDownLatch(1);
-
         /** {@inheritDoc} */
         @Override public Integer load(Integer key) throws CacheLoaderException {
-            if (key == 3)
-                throw new CacheLoaderException();
+            if (key == FAILING_KEY)
+                throw new TestCacheLoaderExpectedException();
 
             return key;
         }
@@ -157,4 +173,14 @@ private static class HangingCacheStore extends CacheStoreAdapter<Integer, Intege
 
         }
     }
+
+    /**
+     *
+     */
+    private static class TestCacheLoaderExpectedException extends CacheLoaderException {
+        /** {@inheritDoc} to reduce amount of logging, trace is not filled */
+        @Override public synchronized Throwable fillInStackTrace() {
+            return this;
+        }
+    }
 }

From 17c62d99a373ecd1dacfbd72cfdf7f4392afb43e Mon Sep 17 00:00:00 2001
From: Alexander Fedotov <alexander.fedotoff@gmail.com>
Date: Fri, 13 Oct 2017 13:39:40 +0300
Subject: [PATCH 401/516] GG-12908 Sequential puts fail for keys implementing
 Externalizable

---
 .../processors/query/GridQueryProcessor.java  | 114 ++---
 .../cache/ExternalizableKeyValueTest.java     | 461 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |   3 +
 3 files changed, 517 insertions(+), 61 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ExternalizableKeyValueTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 76579fa958811..3af4263ab1430 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -246,8 +246,6 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     boolean keyMustDeserialize = mustDeserializeBinary(keyCls);
                     boolean valMustDeserialize = mustDeserializeBinary(valCls);
 
-                    boolean keyOrValMustDeserialize = keyMustDeserialize || valMustDeserialize;
-
                     if (keyCls == null)
                         keyCls = Object.class;
 
@@ -258,15 +256,15 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     desc.tableName(qryEntity.getTableName());
 
                     if (binaryEnabled) {
-                        if (!valMustDeserialize && !SQL_TYPES.contains(valCls))
-                            desc.valueClass(Object.class);
-                        else
+                        if (valMustDeserialize || SQL_TYPES.contains(valCls))
                             desc.valueClass(valCls);
-
-                        if (!keyMustDeserialize && !SQL_TYPES.contains(keyCls))
-                            desc.keyClass(Object.class);
                         else
+                            desc.valueClass(Object.class);
+
+                        if (keyMustDeserialize || SQL_TYPES.contains(keyCls))
                             desc.keyClass(keyCls);
+                        else
+                            desc.keyClass(Object.class);
                     }
                     else {
                         if (keyCls == null)
@@ -284,7 +282,7 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     desc.keyTypeName(qryEntity.getKeyType());
                     desc.valueTypeName(qryEntity.getValueType());
 
-                    if (binaryEnabled && keyOrValMustDeserialize) {
+                    if (binaryEnabled && (keyMustDeserialize || valMustDeserialize)) {
                         if (mustDeserializeClss == null)
                             mustDeserializeClss = new ArrayList<>();
 
@@ -295,17 +293,29 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                             mustDeserializeClss.add(valCls);
                     }
 
+                    processMeta(
+                        qryEntity,
+                        desc,
+                        coCtx,
+                        binaryEnabled && !keyMustDeserialize,
+                        binaryEnabled && !valMustDeserialize
+                    );
+
                     TypeId typeId;
                     TypeId altTypeId = null;
 
                     if (valCls == null || (binaryEnabled && (!valMustDeserialize))) {
-                        processBinaryMeta(qryEntity, desc);
-
                         typeId = new TypeId(ccfg.getName(), ctx.cacheObjects().typeId(qryEntity.getValueType()));
 
                         if (valCls != null)
                             altTypeId = new TypeId(ccfg.getName(), valCls);
+                    }
+                    else {
+                        typeId = new TypeId(ccfg.getName(), valCls);
+                        altTypeId = new TypeId(ccfg.getName(), ctx.cacheObjects().typeId(qryEntity.getValueType()));
+                    }
 
+                    if (binaryEnabled && !keyMustDeserialize) {
                         if (!cctx.customAffinityMapper() && qryEntity.getKeyType() != null) {
                             // Need to setup affinity key for distributed joins.
                             String affField = ctx.cacheObjects().affinityField(qryEntity.getKeyType());
@@ -315,8 +325,6 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                         }
                     }
                     else {
-                        processClassMeta(qryEntity, desc, coCtx);
-
                         AffinityKeyMapper keyMapper = cctx.config().getAffinityMapper();
 
                         if (keyMapper instanceof GridCacheDefaultAffinityKeyMapper) {
@@ -326,9 +334,6 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                             if (affField != null)
                                 desc.affinityKey(affField);
                         }
-
-                        typeId = new TypeId(ccfg.getName(), valCls);
-                        altTypeId = new TypeId(ccfg.getName(), ctx.cacheObjects().typeId(qryEntity.getValueType()));
                     }
 
                     addTypeByName(ccfg, desc);
@@ -361,8 +366,6 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     boolean keyMustDeserialize = mustDeserializeBinary(keyCls);
                     boolean valMustDeserialize = mustDeserializeBinary(valCls);
 
-                    boolean keyOrValMustDeserialize = keyMustDeserialize || valMustDeserialize;
-
                     if (keyCls == null)
                         keyCls = Object.class;
 
@@ -374,15 +377,15 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     desc.name(simpleValType);
 
                     if (binaryEnabled) {
-                        if (!valMustDeserialize && !SQL_TYPES.contains(valCls))
-                            desc.valueClass(Object.class);
-                        else
+                        if (valMustDeserialize || SQL_TYPES.contains(valCls))
                             desc.valueClass(valCls);
-
-                        if (!keyMustDeserialize && !SQL_TYPES.contains(keyCls))
-                            desc.keyClass(Object.class);
                         else
+                            desc.valueClass(Object.class);
+
+                        if (keyMustDeserialize || SQL_TYPES.contains(keyCls))
                             desc.keyClass(keyCls);
+                        else
+                            desc.keyClass(Object.class);
                     }
                     else {
                         desc.valueClass(valCls);
@@ -392,7 +395,7 @@ private void initializeCache(GridCacheContext<?, ?> cctx) throws IgniteCheckedEx
                     desc.keyTypeName(meta.getKeyType());
                     desc.valueTypeName(meta.getValueType());
 
-                    if (binaryEnabled && keyOrValMustDeserialize) {
+                    if (binaryEnabled && (keyMustDeserialize || valMustDeserialize)) {
                         if (mustDeserializeClss == null)
                             mustDeserializeClss = new ArrayList<>();
 
@@ -1461,13 +1464,22 @@ private void processBinaryMeta(CacheTypeMetadata meta, TypeDescriptor d)
     }
 
     /**
-     * Processes declarative metadata for binary object.
+     * Processes declarative metadata.
      *
      * @param qryEntity Declared metadata.
      * @param d Type descriptor.
+     * @param coCtx Cache object context.
+     * @param useBinaryKey Whether a binary key should be used.
+     * @param useBinaryVal Whether a binary value should be used.
      * @throws IgniteCheckedException If failed.
      */
-    private void processBinaryMeta(QueryEntity qryEntity, TypeDescriptor d) throws IgniteCheckedException {
+    private void processMeta(
+        QueryEntity qryEntity,
+        TypeDescriptor d,
+        CacheObjectContext coCtx,
+        boolean useBinaryKey,
+        boolean useBinaryVal
+    ) throws IgniteCheckedException {
         Map<String,String> aliases = qryEntity.getAliases();
 
         if (aliases == null)
@@ -1500,40 +1512,20 @@ private void processBinaryMeta(QueryEntity qryEntity, TypeDescriptor d) throws I
             else
                 isKeyField = (hasKeyFields ? keyFields.contains(entry.getKey()) : null);
 
-            BinaryProperty prop = buildBinaryProperty(entry.getKey(), U.classForName(entry.getValue(), Object.class, true),
-                aliases, isKeyField);
-
-            d.addProperty(prop, false);
-        }
-
-        processIndexes(qryEntity, d);
-    }
-
-    /**
-     * Processes declarative metadata for binary object.
-     *
-     * @param qryEntity Declared metadata.
-     * @param d Type descriptor.
-     * @throws IgniteCheckedException If failed.
-     */
-    private void processClassMeta(
-        QueryEntity qryEntity,
-        TypeDescriptor d,
-        CacheObjectContext coCtx
-    ) throws IgniteCheckedException {
-        Map<String,String> aliases = qryEntity.getAliases();
-
-        if (aliases == null)
-            aliases = Collections.emptyMap();
-
-        for (Map.Entry<String, String> entry : qryEntity.getFields().entrySet()) {
-            ClassProperty prop = buildClassProperty(
-                d.keyClass(),
-                d.valueClass(),
-                entry.getKey(),
-                U.classForName(entry.getValue(), Object.class),
-                aliases,
-                coCtx);
+            boolean isPropBinary = isKeyField != null && isKeyField ? useBinaryKey : useBinaryVal;
+
+            GridQueryProperty prop = isPropBinary ?
+                buildBinaryProperty(
+                    entry.getKey(), U.classForName(entry.getValue(), Object.class, true), aliases, isKeyField
+                ) :
+                buildClassProperty(
+                    d.keyClass(),
+                    d.valueClass(),
+                    entry.getKey(),
+                    U.classForName(entry.getValue(), Object.class),
+                    aliases,
+                    coCtx
+                );
 
             d.addProperty(prop, false);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ExternalizableKeyValueTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ExternalizableKeyValueTest.java
new file mode 100644
index 0000000000000..011553c2e1579
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ExternalizableKeyValueTest.java
@@ -0,0 +1,461 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.io.Serializable;
+import java.util.Objects;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class ExternalizableKeyValueTest extends GridCommonAbstractTest {
+    /**
+     * Key class.
+     */
+    public static class Key implements Serializable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        @QuerySqlField(index = true)
+        protected int key;
+
+        /**
+         * Default constructor.
+         */
+        public Key() {
+            // No-op.
+        }
+
+        /**
+         * Constructor.
+         *
+         * @param key the key
+         */
+        public Key(final int key) {
+            this.key = key;
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public int hashCode() {
+            return Objects.hashCode(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public boolean equals(final Object o) {
+            if (o == this)
+                return true;
+
+            if (o instanceof Key) {
+                final Key that = (Key)o;
+                return key == that.key;
+            }
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "Key{" +
+                "key=" + key +
+                '}';
+        }
+    }
+
+    /**
+     * {@code Externalizable} Key class.
+     */
+    public static class ExternalizableKey extends Key implements Externalizable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Default constructor.
+         */
+        public ExternalizableKey() {
+            // No-op.
+        }
+
+        /**
+         * Constructor
+         *
+         * @param key the key
+         */
+        public ExternalizableKey(int key) {
+            super(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "ExternalizableKey{" +
+                "key=" + key +
+                "}";
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public void writeExternal(final ObjectOutput out)
+            throws IOException {
+            out.writeInt(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException {
+            key = in.readInt();
+        }
+    }
+
+    /**
+     * Value class.
+     */
+    public static class Value implements Serializable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        @QuerySqlField(index = true)
+        protected Integer val;
+
+        /**
+         * Default constructor.
+         */
+        public Value() {
+            // No-op.
+        }
+
+        /**
+         * Constructor.
+         *
+         * @param val Value.
+         */
+        public Value(Integer val) {
+            this.val = val;
+        }
+
+        /**
+         * Returns value.
+         *
+         * @return Value.
+         */
+        public Integer getVal() {
+            return val;
+        }
+
+        /**
+         * Sets value.
+         *
+         * @param val Value.
+         */
+        public void setVal(Integer val) {
+            this.val = val;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "Value{" +
+                "val=" + val +
+                '}';
+        }
+    }
+
+    /**
+     * {@code Externalizable} Value class.
+     */
+    public static class ExternalizableValue extends Value implements Externalizable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Default constructor.
+         */
+        public ExternalizableValue() {
+            // No-op.
+        }
+
+        /**
+         * Constructor.
+         *
+         * @param val Value.
+         */
+        public ExternalizableValue(Integer val) {
+            super(val);
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public void writeExternal(final ObjectOutput out)
+            throws IOException {
+            out.writeInt(val);
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException {
+            setVal(in.readInt());
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "ExternalizableValue{" +
+                "val=" + val +
+                "} ";
+        }
+    }
+
+    /**
+     * Marshaller type.
+     */
+    public enum MarshallerType {
+        /** */
+        JDK,
+        /** */
+        OPTIMIZED,
+        /** */
+        BINARY
+    }
+
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final String CACHE_NAME = "cacheWithExternalizedKeyAndOrValue";
+
+    /**
+     * Returns a marshaller for the specified type.
+     *
+     * @param marshallerType Marshaller type.
+     * @return Marshaller. {@code null} if {@link MarshallerType#BINARY} type
+     * is specified or type is unknown.
+     */
+    private static Marshaller getMarshaller(MarshallerType marshallerType) {
+        switch (marshallerType) {
+            case JDK:
+                return new JdkMarshaller();
+            case OPTIMIZED:
+                return new OptimizedMarshaller();
+        }
+
+        // Binary marshaller is used by default.
+        return null;
+    }
+
+    /** */
+    private MarshallerType marshallerType;
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+        IgniteConfiguration cfg = new IgniteConfiguration()
+            .setGridName(gridName)
+            .setPeerClassLoadingEnabled(false)
+            .setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        Marshaller marshaller = getMarshaller(marshallerType);
+
+        // By default a binary marshaller is used.
+        if (marshaller != null)
+            cfg.setMarshaller(marshaller);
+
+        return cfg;
+    }
+
+    /**
+     * @param keyCls Key class.
+     * @param valCls Val class.
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheConfig(Class<K> keyCls, Class<V> valCls) {
+        return new CacheConfiguration<K, V>(CACHE_NAME).setIndexedTypes(keyCls, valCls);
+    }
+
+    /** */
+    private <K, V> void doSequentialPut(
+        MarshallerType marshallerType, Class<K> keyCls, Class<V> valCls, K key1, V val1, K key2, V val2
+    ) throws Exception {
+        this.marshallerType = marshallerType;
+
+        startGrids(2);
+
+        IgniteCache<K, V> cache = grid(0).createCache(cacheConfig(keyCls, valCls));
+
+        cache.put(key1, val1);
+        cache.put(key2, val2);
+
+        assertEquals(2, cache.size());
+    }
+
+    /** */
+    public void doSequentialPutWithExternalizableKeyAndPrimitiveValue(MarshallerType marshallerType) throws Exception {
+        doSequentialPut(
+            marshallerType,
+            ExternalizableKey.class,
+            Integer.class,
+            new ExternalizableKey(1),
+            1,
+            new ExternalizableKey(2),
+            2
+        );
+    }
+
+    /** */
+    public void doSequentialPutWithExternalizableKeyAndExternalizableValue(MarshallerType marshallerType) throws Exception {
+        doSequentialPut(
+            marshallerType,
+            ExternalizableKey.class,
+            ExternalizableValue.class,
+            new ExternalizableKey(1),
+            new ExternalizableValue(111),
+            new ExternalizableKey(2),
+            new ExternalizableValue(222)
+        );
+    }
+
+    /** */
+    public void doSequentialPutWithExternalizableKeyAndPojoValue(MarshallerType marshallerType) throws Exception {
+        doSequentialPut(
+            marshallerType,
+            ExternalizableKey.class,
+            Value.class,
+            new ExternalizableKey(1),
+            new Value(111),
+            new ExternalizableKey(2),
+            new Value(222)
+        );
+    }
+
+    /** */
+    public void doSequentialPutWithPrimitiveKeyAndExternalizableValue(MarshallerType marshallerType) throws Exception {
+        doSequentialPut(
+            marshallerType,
+            Integer.class,
+            ExternalizableValue.class,
+            1,
+            new ExternalizableValue(111),
+            2,
+            new ExternalizableValue(222)
+        );
+    }
+
+    /** */
+    public void doSequentialPutWithPojoKeyAndExternalizableValue(MarshallerType marshallerType) throws Exception {
+        doSequentialPut(
+            marshallerType,
+            Key.class,
+            ExternalizableValue.class,
+            new Key(1),
+            new ExternalizableValue(111),
+            new Key(2),
+            new ExternalizableValue(222)
+        );
+    }
+
+    /** */
+    public void testSequentialPutWithExternalizableKeyAndPrimitiveValueAndBinaryMarshaller() throws Exception {
+        doSequentialPutWithExternalizableKeyAndPrimitiveValue(MarshallerType.BINARY);
+    }
+
+    /** */
+    public void testSequentialPutWithExternalizableKeyAndPrimitiveValueAndJdkMarshaller() throws Exception {
+        doSequentialPutWithExternalizableKeyAndPrimitiveValue(MarshallerType.JDK);
+    }
+
+    /** */
+    public void testSequentialPutWithExternalizableKeyAndPrimitiveValueAndOptimizedMarshaller() throws Exception {
+        doSequentialPutWithExternalizableKeyAndPrimitiveValue(MarshallerType.OPTIMIZED);
+    }
+
+    /** */
+    public void testSequentialPutWithExternalizableKeyAndExternalizableValueAndBinaryMarshaller() throws Exception {
+        doSequentialPutWithExternalizableKeyAndExternalizableValue(MarshallerType.BINARY);
+    }
+
+    /** */
+    public void testSequentialPutWithExternalizableKeyAndExternalizableValueAndJdkMarshaller() throws Exception {
+        doSequentialPutWithExternalizableKeyAndExternalizableValue(MarshallerType.JDK);
+    }
+
+    /** */
+    public void testSequentialPutWithExternalizableKeyAndExternalizableValueAndOptimizedMarshaller() throws Exception {
+        doSequentialPutWithExternalizableKeyAndExternalizableValue(MarshallerType.OPTIMIZED);
+    }
+
+    /** */
+    public void testSequentialPutWithExternalizableKeyAndPojoValueAndBinaryMarshaller() throws Exception {
+        doSequentialPutWithExternalizableKeyAndPojoValue(MarshallerType.BINARY);
+    }
+
+    /** */
+    public void testSequentialPutWithExternalizableKeyAndPojoValueAndJdkMarshaller() throws Exception {
+        doSequentialPutWithExternalizableKeyAndPojoValue(MarshallerType.JDK);
+    }
+
+    /** */
+    public void testSequentialPutWithExternalizableKeyAndPojoValueAndOptimizedMarshaller() throws Exception {
+        doSequentialPutWithExternalizableKeyAndPojoValue(MarshallerType.OPTIMIZED);
+    }
+
+    /** */
+    public void testSequentialPutWithPrimitiveKeyAndExternalizableValueAndBinaryMarshaller() throws Exception {
+        doSequentialPutWithPrimitiveKeyAndExternalizableValue(MarshallerType.BINARY);
+    }
+
+    /** */
+    public void testSequentialPutWithPrimitiveKeyAndExternalizableValueAndJdkMarshaller() throws Exception {
+        doSequentialPutWithPrimitiveKeyAndExternalizableValue(MarshallerType.JDK);
+    }
+
+    /** */
+    public void testSequentialPutWithPrimitiveKeyAndExternalizableValueAndOptimizedMarshaller() throws Exception {
+        doSequentialPutWithPrimitiveKeyAndExternalizableValue(MarshallerType.OPTIMIZED);
+    }
+
+    /** */
+    public void testSequentialPutWithPojoKeyAndExternalizableValueAndBinaryMarshaller() throws Exception {
+        doSequentialPutWithPojoKeyAndExternalizableValue(MarshallerType.BINARY);
+    }
+
+    /** */
+    public void testSequentialPutWithPojoKeyAndExternalizableValueAndJdkMarshaller() throws Exception {
+        doSequentialPutWithPojoKeyAndExternalizableValue(MarshallerType.JDK);
+    }
+
+    /** */
+    public void testSequentialPutWithPojoKeyAndExternalizableValueAndOptimizedMarshaller() throws Exception {
+        doSequentialPutWithPojoKeyAndExternalizableValue(MarshallerType.OPTIMIZED);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index bfcc374ff3fec..160f85d0a1387 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.internal.processors.cache.CacheRebalancingSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheSerializableTransactionsTest;
 import org.apache.ignite.internal.processors.cache.EntryVersionConsistencyReadThroughTest;
+import org.apache.ignite.internal.processors.cache.ExternalizableKeyValueTest;
 import org.apache.ignite.internal.processors.cache.GridCacheOffHeapCleanupTest;
 import org.apache.ignite.internal.processors.cache.GridCacheSwapSpaceSpiConsistencySelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCachePutStackOverflowSelfTest;
@@ -72,6 +73,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(GridCachePartitionEvictionDuringReadThroughSelfTest.class);
 
+        suite.addTestSuite(ExternalizableKeyValueTest.class);
+
         return suite;
     }
 }

From 11e3d2c39adcbe779c41316877903500c611921c Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 13 Oct 2017 18:12:55 +0300
Subject: [PATCH 402/516] GG-12941: Mute test
 CacheLateAffinityAssignmentTest.testNoForceKeysRequests

---
 .../cache/distributed/CacheLateAffinityAssignmentTest.java      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 761a7f0979a1d..4dbcd327a9437 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -1768,6 +1768,8 @@ public void testServiceReassign() throws Exception {
      * @throws Exception If failed.
      */
     public void testNoForceKeysRequests() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-5510");
+
         cacheC = new IgniteClosure<String, CacheConfiguration[]>() {
             @Override public CacheConfiguration[] apply(String s) {
                 return null;

From 5c35a58788807be8f1b7d79e6174b27f81d7a1f6 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 13 Oct 2017 18:25:50 +0300
Subject: [PATCH 403/516] GG-12941: Fixed hanged test
 CacheLateAffinityAssignmentTest.testJoinExchangeBecomeCoordinator.

---
 .../cache/distributed/CacheLateAffinityAssignmentTest.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 4dbcd327a9437..cf32c353d4d55 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -1103,7 +1103,7 @@ public void testJoinExchangeBecomeCoordinator() throws Exception {
                 U.sleep(5000);
 
                 for (int i = 0; i < NODES; i++)
-                    stopGrid(i);
+                    stopGrid(i, false, false);
 
                 return null;
             }

From 5b50c724ccb3732007416627a554252e781290a6 Mon Sep 17 00:00:00 2001
From: Konstantin Dudkov <kdudkov@ya.ru>
Date: Tue, 17 Oct 2017 11:43:30 +0300
Subject: [PATCH 404/516] IGNITE-6581 fix client deadlock

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |   3 +-
 .../TcpDiscoveryClientConnectionFailTest.java | 161 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   3 +
 3 files changed, 166 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientConnectionFailTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 6c12ceeb364bc..ce4f1b7fe4b9a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -1704,7 +1704,8 @@ else if (msg == SPI_RECONNECT_FAILED) {
 
                                 onDisconnected();
 
-                                notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
+                                if (joinLatch.getCount() == 0)
+                                    notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
                             }
 
                             UUID newId = UUID.randomUUID();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientConnectionFailTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientConnectionFailTest.java
new file mode 100644
index 0000000000000..c77f1b7f62b45
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientConnectionFailTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.Socket;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryClientHeartbeatMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddedMessage;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class TcpDiscoveryClientConnectionFailTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+    /** */
+    private static volatile boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        if (gridName.endsWith("0"))
+            cfg.setFailureDetectionTimeout(100);
+
+        TestDiscoverySpi spi = new TestDiscoverySpi(gridName);
+        spi.setClientReconnectDisabled(false);
+        spi.setIpFinder(ipFinder);
+
+        spi.setJoinTimeout(20000);
+        spi.setMaxMissedClientHeartbeats(2);
+        spi.failureDetectionTimeoutEnabled(true);
+
+        cfg.setDiscoverySpi(spi);
+
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /** */
+    public void testClientFailsIfCantConnect() throws Exception {
+        client = false;
+
+        startGrid(0);
+
+        client = true;
+
+        IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    startGrid(1);
+                }
+                catch (Exception ignored) {
+                }
+
+            }
+        }, 1, "client start thread");
+
+        try {
+            fut.get(10000);
+        }
+        catch (IgniteFutureTimeoutCheckedException e) {
+            fail("client node start hangs (possible deadlock detected)");
+        }
+
+    }
+
+    /**
+     *
+     */
+    private static class TestDiscoverySpi extends TcpDiscoverySpi {
+        /** */
+        private final String name;
+        /** */
+        private boolean block;
+
+        /** */
+        TestDiscoverySpi(String name) {
+            this.name = name;
+        }
+
+        /**
+         * @param msg Message.
+         * @return {@code False} if should not further process message.
+         * @throws IOException If failed.
+         */
+        private boolean onMessage(TcpDiscoveryAbstractMessage msg) throws IOException {
+            if (msg == null)
+                return true;
+
+            boolean blocked = false;
+
+            // we block first connection attempt only
+            if (msg instanceof TcpDiscoveryNodeAddedMessage)
+                block = !block;
+
+            if (msg instanceof TcpDiscoveryClientHeartbeatMessage && block)
+                blocked = true;
+
+            if (msg instanceof TcpDiscoveryNodeAddFinishedMessage && block)
+                blocked = true;
+
+            return !blocked;
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, byte[] data,
+            long timeout) throws IOException {
+
+            if (onMessage(msg))
+                super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(TcpDiscoveryAbstractMessage msg, Socket sock, int res,
+            long timeout) throws IOException {
+            if (onMessage(msg))
+                super.writeToSocket(msg, sock, res, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (onMessage(msg))
+                super.writeToSocket(sock, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (onMessage(msg))
+                super.writeToSocket(sock, out, msg, timeout);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index c506ca7e95bf2..aa9ac93078947 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryClientConnectionFailTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMultiThreadedTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeAttributesUpdateOnReconnectTest;
@@ -96,6 +97,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(IgniteClientConnectTest.class));
         suite.addTest(new TestSuite(IgniteClientReconnectMassiveShutdownTest.class));
 
+        suite.addTest(new TestSuite(TcpDiscoveryClientConnectionFailTest.class));
+
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));

From 0ed98c02d3253d3dafe41a1b4f2d148b8cc0eb50 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Tue, 17 Oct 2017 18:19:11 +0300
Subject: [PATCH 405/516] GG-12881 Possible live-lock in discovery worker.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../GridDhtPartitionsExchangeFuture.java      | 78 +++++++++++++++----
 1 file changed, 61 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 13fdb30137f58..3578af40c1f36 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.GridPlainRunnable;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -212,6 +213,13 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** Cache change requests. */
     private ExchangeActions exchActions;
 
+    /** Disco thread flag. */
+    private final ThreadLocal<Boolean> discoThread = new ThreadLocal<Boolean>(){
+        @Override protected Boolean initialValue() {
+            return false;
+        }
+    };
+
     /**
      * Dummy future created to trigger reassignments if partition topology changed while preloading.
      *
@@ -1072,6 +1080,30 @@ private void sendPartitions(ClusterNode oldestNode) {
         }
     }
 
+    /**
+     * @param nodeIds Node ids.
+     * @param req Partition single request.
+     */
+    private void sendPartitionRequest(Set<UUID> nodeIds, GridDhtPartitionsSingleRequest req) {
+        for (UUID nodeId : nodeIds) {
+            if (log.isDebugEnabled())
+                log.debug("Sending message: [node=" + nodeId + ", req=" + req + ']');
+
+            try {
+                // It is possible that some nodes finished exchange with previous coordinator.
+                cctx.io().send(nodeId, req, SYSTEM_POOL);
+            }
+            catch (ClusterTopologyCheckedException ignored) {
+                if (log.isDebugEnabled())
+                    log.debug("Node left during partition exchange [nodeId=" + nodeId +
+                        ", exchId=" + exchId + ']');
+            }
+            catch (IgniteCheckedException e) {
+                U.error(log, "Failed to request partitions from node: " + nodeId, e);
+            }
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public boolean onDone(@Nullable AffinityTopologyVersion res, @Nullable Throwable err) {
         boolean realExchange = !dummy && !forcePreload;
@@ -1742,7 +1774,11 @@ private void onDiscoveryEvent(IgniteRunnable c) {
             assert discoEvts.isEmpty() : discoEvts;
         }
 
+        discoThread.set(true);
+
         c.run();
+
+        discoThread.set(false);
     }
 
     /**
@@ -1785,6 +1821,9 @@ public void onNodeLeft(final ClusterNode node) {
         try {
             onDiscoveryEvent(new IgniteRunnable() {
                 @Override public void run() {
+                    // true if method was invoke from disco thread, otherwise false.
+                    boolean discoThread = GridDhtPartitionsExchangeFuture.this.discoThread.get();
+
                     if (isDone() || !enterBusy())
                         return;
 
@@ -1793,7 +1832,7 @@ public void onNodeLeft(final ClusterNode node) {
                         boolean allReceived = false;
                         Set<UUID> reqFrom = null;
 
-                        ClusterNode crd0;
+                        final ClusterNode crd0;
 
                         discoCache.updateAlives(node);
 
@@ -1849,22 +1888,19 @@ public void onNodeLeft(final ClusterNode node) {
                             }
 
                             if (crdChanged && reqFrom != null) {
-                                GridDhtPartitionsSingleRequest req = new GridDhtPartitionsSingleRequest(exchId);
-
-                                for (UUID nodeId : reqFrom) {
-                                    try {
-                                        // It is possible that some nodes finished exchange with previous coordinator.
-                                        cctx.io().send(nodeId, req, SYSTEM_POOL);
-                                    }
-                                    catch (ClusterTopologyCheckedException ignored) {
-                                        if (log.isDebugEnabled())
-                                            log.debug("Node left during partition exchange [nodeId=" + nodeId +
-                                                ", exchId=" + exchId + ']');
-                                    }
-                                    catch (IgniteCheckedException e) {
-                                        U.error(log, "Failed to request partitions from node: " + nodeId, e);
-                                    }
+                                final GridDhtPartitionsSingleRequest req = new GridDhtPartitionsSingleRequest(exchId);
+
+                                if (discoThread) {
+                                    final Set<UUID> reqFrom0 = reqFrom;
+
+                                    cctx.kernalContext().closure().runLocalSafe(new GridPlainRunnable() {
+                                        @Override public void run() {
+                                            sendPartitionRequest(reqFrom0, req);
+                                        }
+                                    }, true);
                                 }
+                                else
+                                    sendPartitionRequest(reqFrom, req);
                             }
 
                             for (Map.Entry<ClusterNode, GridDhtPartitionsSingleMessage> m : singleMsgs.entrySet())
@@ -1872,7 +1908,15 @@ public void onNodeLeft(final ClusterNode node) {
                         }
                         else {
                             if (crdChanged) {
-                                sendPartitions(crd0);
+                                if (discoThread) {
+                                    cctx.kernalContext().closure().runLocalSafe(new GridPlainRunnable() {
+                                        @Override public void run() {
+                                            sendPartitions(crd0);
+                                        }
+                                    }, true);
+                                }
+                                else
+                                    sendPartitions(crd0);
 
                                 for (Map.Entry<ClusterNode, GridDhtPartitionsFullMessage> m : fullMsgs.entrySet())
                                     processMessage(m.getKey(), m.getValue());

From 656d458bf3e1e70accf7c56b650ffbdc2f0ab6ea Mon Sep 17 00:00:00 2001
From: vsisko <vsisko@gridgain.com>
Date: Wed, 18 Oct 2017 20:39:13 +0700
Subject: [PATCH 406/516] WC-322 Annotations for REST serialization.

---
 .../apache/ignite/internal/visor/file/VisorFileBlockTask.java  | 3 ++-
 .../org/apache/ignite/internal/visor/log/VisorLogFile.java     | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlockTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlockTask.java
index 58a25eed8b673..8b73d20cec954 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlockTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlockTask.java
@@ -23,6 +23,7 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.NoSuchFileException;
+import org.apache.ignite.internal.LessNamingBean;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -50,7 +51,7 @@ public class VisorFileBlockTask extends VisorOneNodeTask<VisorFileBlockTask.Viso
      * Arguments for {@link VisorFileBlockTask}
      */
     @SuppressWarnings("PublicInnerClass")
-    public static class VisorFileBlockArg implements Serializable {
+    public static class VisorFileBlockArg implements Serializable, LessNamingBean {
         /** */
         private static final long serialVersionUID = 0L;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogFile.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogFile.java
index 211a171a5fbe8..59bd3d2c0222d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogFile.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogFile.java
@@ -19,12 +19,13 @@
 
 import java.io.File;
 import java.io.Serializable;
+import org.apache.ignite.internal.LessNamingBean;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
  * Visor log file.
  */
-public class VisorLogFile implements Serializable {
+public class VisorLogFile implements Serializable, LessNamingBean {
     /** */
     private static final long serialVersionUID = 0L;
 

From c709db2ea0185cd005a138b4c388ee1f97315115 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Mon, 23 Oct 2017 12:50:15 +0300
Subject: [PATCH 407/516] IGNITE-6071 White list of exceptions to suppress in
 createTcpClient.

---
 .../tcp/TcpCommunicationSpi.java              |   3 +-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |   2 +
 ...tConnectAfterCommunicationFailureTest.java | 156 ++++++++++++++++++
 .../IgniteClientReconnectTestSuite.java       |   4 +-
 4 files changed, 163 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteClientConnectAfterCommunicationFailureTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index de1f76d9f1b61..dffa181058c56 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3144,7 +3144,8 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                 }
             }
 
-            if (X.hasCause(errs, ConnectException.class, HandshakeException.class))
+            if (!X.hasCause(errs, SocketTimeoutException.class, HandshakeTimeoutException.class,
+                IgniteSpiOperationTimeoutException.class))
                 throw errs;
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 7816458407a11..640dcec85b0b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -724,6 +724,8 @@ else if (!spi.failureDetectionTimeoutEnabled() && reconCnt == spi.getReconnectCo
                     finally {
                         U.closeQuiet(sock);
                     }
+
+                    U.sleep(200);
                 }
             }
             catch (Throwable t) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientConnectAfterCommunicationFailureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientConnectAfterCommunicationFailureTest.java
new file mode 100644
index 0000000000000..301d5f24e4922
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientConnectAfterCommunicationFailureTest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Arrays;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests client to be able restore connection to cluster on subsequent attempts after communication problems.
+ */
+public class IgniteClientConnectAfterCommunicationFailureTest extends GridCommonAbstractTest {
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setNetworkTimeout(500);
+        cfg.setCommunicationSpi(new TcpCommunicationSpi(gridName.contains("block")));
+
+        if (gridName.contains("client")) {
+            cfg.setClientMode(true);
+        }
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientReconnects() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+        startGrid("client-block");
+
+        assertEquals(1, srv2.cluster().forClients().nodes().size());
+        assertEquals(1, srv1.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientThreadsSuspended() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+        Ignite client = startGrid("client");
+
+        boolean blockedAnything = false;
+
+        for (Thread thread : Thread.getAllStackTraces().keySet()) {
+            if (thread.getName().contains("%client%")) {
+                thread.suspend();
+                blockedAnything = true;
+            }
+        }
+
+        Thread.sleep(10000);
+
+        for (Thread thread : Thread.getAllStackTraces().keySet()) {
+            if (thread.getName().contains("%client%"))
+                thread.resume();
+        }
+
+        for (int j = 0; j < 10; j++) {
+            boolean topOk = true;
+
+            for (Ignite node : Arrays.asList(srv1, srv2, client)) {
+                if (node.cluster().nodes().size() != 3) {
+                    U.warn(log, "Grid size is incorrect (will re-run check in 1000 ms) " +
+                        "[name=" + node.name() + ", size=" + node.cluster().nodes().size() + ']');
+
+                    topOk = false;
+
+                    break;
+                }
+            }
+
+            if (topOk)
+                return;
+            else
+                Thread.sleep(1000);
+        }
+
+        assertTrue(blockedAnything);
+        assertEquals(1, srv2.cluster().forClients().nodes().size());
+        assertEquals(1, srv1.cluster().forClients().nodes().size());
+    }
+
+    /**
+     * Will never connect with the first node id, normal operation after.
+     */
+    private class TcpCommunicationSpi extends org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi {
+        /**
+         * Whether this instance should actually block.
+         */
+        private final boolean isBlocking;
+
+        /**
+         * Local node ID that is prevented from creating connections.
+         */
+        private volatile UUID blockedNodeId = null;
+
+        /**
+         *
+         * @param isBlocking Whether this instance should actually block.
+         */
+        public TcpCommunicationSpi(boolean isBlocking) {
+            this.isBlocking = isBlocking;
+        }
+
+        /** {@inheritDoc} */
+        @Override protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
+            throws IgniteCheckedException {
+            if (blockHandshakeOnce(getLocalNode().id())) {
+                throw new IgniteCheckedException("Node is blocked");
+            }
+
+            return super.createTcpClient(node, connIdx);
+        }
+
+        /** Check if this connection is blocked. */
+        private boolean blockHandshakeOnce(UUID nodeId) {
+            if (isBlocking && (blockedNodeId == null || blockedNodeId.equals(nodeId))) {
+                blockedNodeId = nodeId;
+                return true;
+            }
+            return false;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
index 03d3fe230c296..d0e907cde01f8 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteClientReconnectTestSuite.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.IgniteClientConnectAfterCommunicationFailureTest;
 import org.apache.ignite.internal.IgniteClientReconnectApiExceptionTest;
 import org.apache.ignite.internal.IgniteClientReconnectAtomicsTest;
 import org.apache.ignite.internal.IgniteClientReconnectBinaryContexTest;
@@ -43,6 +44,7 @@ public class IgniteClientReconnectTestSuite extends TestSuite {
     public static TestSuite suite() throws Exception {
         TestSuite suite = new TestSuite("Ignite Client Reconnect Test Suite");
 
+        suite.addTestSuite(IgniteClientConnectAfterCommunicationFailureTest.class);
         suite.addTestSuite(IgniteClientReconnectStopTest.class);
         suite.addTestSuite(IgniteClientReconnectApiExceptionTest.class);
         suite.addTestSuite(IgniteClientReconnectDiscoveryStateTest.class);
@@ -59,4 +61,4 @@ public static TestSuite suite() throws Exception {
 
         return suite;
     }
-}
\ No newline at end of file
+}

From d39e17fd7af5674a17ef9e81bc5936baabc782dc Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 27 Oct 2017 11:31:45 +0300
Subject: [PATCH 408/516] GG-12887 - Add custom cache validator.

---
 .../apache/ignite/IgniteSystemProperties.java |   6 +
 .../processors/cache/GridCacheAdapter.java    |  35 ++
 .../GridCachePartitionExchangeManager.java    |  13 +
 .../processors/cache/GridCacheProcessor.java  |  33 ++
 .../processors/cache/IgniteCacheProxy.java    |   7 +
 .../dht/GridPartitionedGetFuture.java         |  10 +
 .../dht/GridPartitionedSingleGetFuture.java   |  10 +
 .../dht/atomic/GridDhtAtomicCache.java        |  13 +
 .../GridDhtPartitionsExchangeFuture.java      |  57 +-
 .../distributed/near/GridNearGetFuture.java   |   8 +
 .../GridCacheDistributedQueryManager.java     |  11 +
 .../cache/transactions/IgniteTxHandler.java   |  66 +++
 .../cluster/GridUpdateNotifier.java           |   6 -
 ...iteTopologyValidatorAbstractCacheTest.java |   9 +-
 ...eTopologyValidatorAbstractTxCacheTest.java |  10 +-
 .../GridLostPartitionRebalanceTest.java       |  11 +-
 .../GridLostPartitionValidationTest.java      | 536 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite3.java     |   2 +
 .../processors/query/h2/IgniteH2Indexing.java |  37 +-
 .../query/h2/sql/GridSqlQueryParser.java      |  45 ++
 .../h2/twostep/GridMapQueryExecutor.java      |  50 +-
 .../GridLostPartitionValidationQueryTest.java | 300 ++++++++++
 .../IgniteCacheWithIndexingTestSuite.java     |   3 +
 23 files changed, 1250 insertions(+), 28 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionValidationTest.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridLostPartitionValidationQueryTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index f570e1246aa1d..3c5a5a6c5d44b 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Properties;
 import javax.net.ssl.HostnameVerifier;
+import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.jetbrains.annotations.Nullable;
@@ -600,6 +601,11 @@ public final class IgniteSystemProperties {
     /** Ignite marshaller cache reread pause. */
     public static final String IGNITE_MARSHALLER_CACHE_REREAD_PAUSE = "IGNITE_MARSHALLER_CACHE_REREAD_PAUSE";
 
+    /** Class name of the closure {@link IgniteClosure<String, Throwable>} that
+     * will be invoked once per topology and validates cache. If it returns or throws exception,
+     * that means cache is invalid.*/
+    public static final String IGNITE_CACHE_VALIDATOR = "IGNITE_CACHE_VALIDATOR";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 2fc6359c33b5b..ca1b8c90163d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -86,6 +86,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter;
@@ -728,6 +729,8 @@ else if (modes.heap) {
         if (modes.primary || modes.backup) {
             AffinityTopologyVersion topVer = ctx.affinity().affinityTopologyVersion();
 
+            validateCache(topVer);
+
             GridCacheSwapManager swapMgr = ctx.isNear() ? ctx.near().dht().context().swap() : ctx.swap();
 
             if (modes.swap)
@@ -773,6 +776,8 @@ public String toString() {
             if (!ctx.isLocal()) {
                 AffinityTopologyVersion topVer = ctx.affinity().affinityTopologyVersion();
 
+                validateCache(topVer);
+
                 int part = ctx.affinity().partition(cacheKey);
 
                 boolean nearKey;
@@ -853,6 +858,24 @@ public String toString() {
         }
     }
 
+    /**
+     * @throws IgniteCheckedException If validation failed.
+     */
+    private void validateCache(AffinityTopologyVersion topVer) throws IgniteCheckedException {
+        if (isLocal())
+            return;
+
+        GridDhtTopologyFuture fut = ctx.shared().exchange().exchangeFuture(topVer);
+
+        if (fut == null)
+            fut = ctx.topologyVersionFuture();
+
+        Throwable exc = fut.validateCache(ctx);
+
+        if (exc != null)
+            throw new IgniteCheckedException(exc);
+    }
+
     /**
      * @param key Key.
      * @param heap Read heap flag.
@@ -1873,6 +1896,18 @@ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
 
                 int keysSize = keys.size();
 
+                if (!isLocal()) {
+                    GridDhtTopologyFuture fut = ctx.shared().exchange().exchangeFuture(topVer);
+
+                    if (fut == null)
+                        fut = ctx.topologyVersionFuture();
+
+                    Throwable exc = fut.validateCache(ctx);
+
+                    if (exc != null)
+                        return new GridFinishedFuture<>(exc);
+                }
+
                 final Map<K1, V1> map = keysSize == 1 ?
                     (Map<K1, V1>)new IgniteBiTuple<>() :
                     U.<K1, V1>newHashMap(keysSize);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index a41004bc771c7..183debd4148d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -698,6 +698,19 @@ public List<GridDhtPartitionsExchangeFuture> exchangeFutures() {
         return exchFuts.values();
     }
 
+    /**
+     * @param topVer Topology version.
+     * @return Exchange future.
+     */
+    @Nullable public GridDhtPartitionsExchangeFuture exchangeFuture(AffinityTopologyVersion topVer) {
+        for (GridDhtPartitionsExchangeFuture fut : exchFuts.values()) {
+            if (fut.topologyVersion().equals(topVer))
+                return fut;
+        }
+
+        return null;
+    }
+
     /**
      * @return {@code True} if pending future queue is empty.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 488cc2dd37e23..8a041b2a1a4ab 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -42,6 +42,7 @@
 import javax.management.MBeanServer;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicWriteOrderMode;
 import org.apache.ignite.cache.CacheExistsException;
 import org.apache.ignite.cache.CacheMemoryMode;
@@ -123,6 +124,7 @@
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_REMOVED_ENTRIES_TTL;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_VALIDATOR;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
 import static org.apache.ignite.IgniteSystemProperties.getBoolean;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -199,6 +201,9 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     /** */
     private Map<UUID, DynamicCacheChangeBatch> clientReconnectReqs;
 
+    /** */
+    private IgniteClosure<String, Throwable> validator;
+
     /**
      * @param ctx Kernal context.
      */
@@ -319,6 +324,27 @@ else if (cfg.getWriteSynchronizationMode() != FULL_SYNC &&
                 throw new IgniteCheckedException("Cannot set both cache writer factory and cache store factory " +
                     "for cache: " + U.maskName(cfg.getName()));
         }
+
+        try {
+            String validatorClsName = IgniteSystemProperties.getString(IGNITE_CACHE_VALIDATOR, null);
+
+            if (validatorClsName != null) {
+                ClassLoader ldr = U.resolveClassLoader(cacheObjCtx.kernalContext().config());
+
+                Class<?> validatorCls = ldr.loadClass(validatorClsName);
+
+                if (IgniteClosure.class.isAssignableFrom(validatorCls))
+                    validator = (IgniteClosure<String, Throwable>)validatorCls.newInstance();
+
+                if (validator != null)
+                    cacheObjCtx.kernalContext().resource().injectGeneric(validator);
+            }
+        }
+        catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
+            throw new IgniteCheckedException("Unable to instantiate cache validator defined by "
+                + IGNITE_CACHE_VALIDATOR + " system property", e);
+        }
+
     }
 
     /**
@@ -3868,6 +3894,13 @@ private static String unmaskNull(String name) {
         return name == NULL_NAME ? null : name;
     }
 
+    /**
+     * Cache validator.
+     */
+    public IgniteClosure<String, Throwable> cacheValidator() {
+        return validator;
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
index f806d05ab919c..2234c33c430c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
@@ -855,6 +855,13 @@ private void validate(Query qry) {
             (qry instanceof SqlQuery || qry instanceof SqlFieldsQuery || qry instanceof TextQuery))
             throw new CacheException("Failed to execute query. Add module 'ignite-indexing' to the classpath " +
                 "of all Ignite nodes.");
+
+        if (!ctx.isLocal()) {
+            Throwable exc = ctx.topologyVersionFuture().validateCache(ctx);
+
+            if (exc != null)
+                throw new CacheException(exc);
+        }
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 60688cc8c0fb2..4dcb300e92fb1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -245,6 +245,16 @@ private void map(
             return;
         }
 
+        GridDhtTopologyFuture exchFut = cctx.topologyVersionFuture();
+
+        Throwable exc = exchFut.validateCache(cctx);
+
+        if (exc != null) {
+            onDone(exc);
+
+            return;
+        }
+
         Map<ClusterNode, LinkedHashMap<KeyCacheObject, Boolean>> mappings = U.newHashMap(cacheNodes.size());
 
         final int keysSize = keys.size();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index 2ed5c60128e30..b667a10f889d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -207,6 +207,16 @@ public void init() {
      */
     @SuppressWarnings("unchecked")
     private void map(AffinityTopologyVersion topVer) {
+        GridDhtTopologyFuture exchFut = cctx.topologyVersionFuture();
+
+        Throwable exc = exchFut.validateCache(cctx);
+
+        if (exc != null) {
+            onDone(exc);
+
+            return;
+        }
+
         ClusterNode node = mapKeyToNode(topVer);
 
         if (node == null) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index ad16f1f52fc74..574a5677671cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -67,6 +67,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedGetFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedSingleGetFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
@@ -1851,6 +1852,18 @@ private void updateAllAsyncInternal0(
         IgniteCacheExpiryPolicy expiry = null;
 
         try {
+            if (!isLocal()) {
+                GridDhtTopologyFuture fut = ctx.shared().exchange().exchangeFuture(req.topologyVersion());
+
+                if (fut == null)
+                    fut = ctx.topologyVersionFuture();
+
+                Throwable exc = fut.validateCache(ctx);
+
+                if (exc != null)
+                    throw new IgniteException(exc);
+            }
+
             // If batch store update is enabled, we need to lock all entries.
             // First, need to acquire locks on cache entries, then check filter.
             List<GridDhtCacheEntry> locked = lockEntries(req, req.topologyVersion());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 3578af40c1f36..59084f542566c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -64,6 +64,7 @@
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
+import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridPlainRunnable;
@@ -76,6 +77,7 @@
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteRunnable;
 import org.jetbrains.annotations.Nullable;
@@ -186,6 +188,9 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** Cache validation results. */
     private volatile Map<Integer, Boolean> cacheValidRes;
 
+    /** Custom cache validation results. */
+    private volatile Map<Integer, Throwable> customCacheValidRes;
+
     /** Skip preload flag. */
     private boolean skipPreload;
 
@@ -471,6 +476,15 @@ public void init() throws IgniteInterruptedCheckedException {
 
             skipPreload = cctx.kernalContext().clientNode();
 
+            if (log.isInfoEnabled()) {
+                log.info("Started exchange init [topVer=" + topologyVersion() +
+                    ", crd=" + crdNode +
+                    ", evt=" + IgniteUtils.gridEventName(discoEvt.type()) +
+                    ", evtNode=" + discoEvt.eventNode().id() +
+                    ", customEvt=" + (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT
+                    ? ((DiscoveryCustomEvent)discoEvt).customMessage() : null) + ']');
+            }
+
             ExchangeType exchange;
 
             Collection<DynamicCacheDescriptor> receivedCaches;
@@ -1138,17 +1152,41 @@ private void sendPartitionRequest(Set<UUID> nodeIds, GridDhtPartitionsSingleRequ
                 detectLostPartitions();
 
             Map<Integer, Boolean> m = null;
+            Map<Integer, Throwable> m2 = null;
+
+            IgniteClosure<String, Throwable> validator = cctx.cache().cacheValidator();
 
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
-                if (cacheCtx.config().getTopologyValidator() != null && !CU.isSystemCache(cacheCtx.name())) {
+                if (CU.isSystemCache(cacheCtx.name()))
+                    continue;
+
+                if (cacheCtx.config().getTopologyValidator() != null) {
                     if (m == null)
                         m = new HashMap<>();
 
                     m.put(cacheCtx.cacheId(), cacheCtx.config().getTopologyValidator().validate(discoEvt.topologyNodes()));
                 }
+
+                if (validator != null) {
+                    if (m2 == null)
+                        m2 = new HashMap<>();
+
+                    Throwable t;
+
+                    try {
+                        t = validator.apply(cacheCtx.name());
+                    }
+                    catch (Throwable e) {
+                        t = e;
+                    }
+
+                    if (t != null)
+                        m2.put(cacheCtx.cacheId(), t);
+                }
             }
 
             cacheValidRes = m != null ? m : Collections.<Integer, Boolean>emptyMap();
+            customCacheValidRes = m2 != null ? m2 : Collections.<Integer, Throwable>emptyMap();
         }
 
         cctx.exchange().onExchangeDone(this, err);
@@ -1197,6 +1235,23 @@ private void sendPartitionRequest(Set<UUID> nodeIds, GridDhtPartitionsSingleRequ
             }
         }
 
+        if (cctx.kernalContext().cache().cacheValidator() != null) {
+            Throwable t;
+
+            Map<Integer, Throwable> resMap = customCacheValidRes;
+
+            if (resMap != null)
+                t = resMap.get(cctx.cacheId());
+            else
+                // Do not cache results, because it's better to do on exchange done.
+                t = cctx.kernalContext().cache().cacheValidator().apply(cctx.name());
+
+            if (t != null) {
+                return new IgniteCheckedException("Failed to perform cache operation " +
+                    "(cache is not valid): " + cctx.name(), t);
+            }
+        }
+
         return null;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index a318724c70ae1..b061593b109fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -256,6 +256,14 @@ private void map(
         Map<ClusterNode, LinkedHashMap<KeyCacheObject, Boolean>> mapped,
         final AffinityTopologyVersion topVer
     ) {
+        Throwable exc = cctx.topologyVersionFuture().validateCache(cctx);
+
+        if (exc != null) {
+            onDone(exc);
+
+            return;
+        }
+
         Collection<ClusterNode> affNodes = CU.affinityNodes(cctx, topVer);
 
         if (affNodes.isEmpty()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
index eb5e214b89474..8457180e7b3a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
 import org.apache.ignite.internal.util.GridBoundedConcurrentOrderedSet;
 import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
@@ -219,6 +220,16 @@ protected void removeQueryFuture(long reqId) {
                     threads.put(req.id(), Thread.currentThread());
 
                     try {
+                        GridDhtTopologyFuture fut = cctx.shared().exchange().exchangeFuture(req.topologyVersion());
+
+                        if (fut == null)
+                            fut = cctx.topologyVersionFuture();
+
+                        Throwable exc = fut.validateCache(cctx);
+
+                        if (exc != null)
+                            throw new IgniteCheckedException(exc);
+
                         GridCacheQueryInfo info = distributedQueryInfo(sndId, req);
 
                         if (info == null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index bfbd86cba5acf..9c7dfd579309c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -18,8 +18,11 @@
 package org.apache.ignite.internal.processors.cache.transactions;
 
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.UUID;
+import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
@@ -38,9 +41,11 @@
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryFuture;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryRequest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryResponse;
+import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxPrepareRequest;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxRemoteAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishResponse;
@@ -120,6 +125,40 @@ public IgniteInternalFuture<?> processNearTxPrepareRequest(final UUID nearNodeId
                 ", node=" + nearNodeId + ']');
         }
 
+        if (ctx.cache().cacheValidator() != null) {
+            for (GridCacheContext cctx : caches(req)) {
+                GridDhtTopologyFuture fut = ctx.exchange().exchangeFuture(req.topologyVersion());
+
+                if (fut == null)
+                    fut = cctx.topologyVersionFuture();
+
+                Throwable exc = fut.validateCache(cctx);
+
+                if (exc != null) {
+                    GridNearTxPrepareResponse resp = new GridNearTxPrepareResponse(
+                        req.version(),
+                        req.futureId(),
+                        req.miniId(),
+                        req.version(),
+                        req.version(),
+                        null,
+                        new CacheException(exc),
+                        null,
+                        req.deployInfo() != null
+                    );
+
+                    try {
+                        ctx.io().send(nearNodeId, resp, req.policy());
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Failed to prepare DHT transaction: [req=" + req + ']', e);
+                    }
+
+                    return new GridFinishedFuture<>(resp);
+                }
+            }
+        }
+
         IgniteInternalFuture<GridNearTxPrepareResponse> fut = prepareTx(nearNodeId, null, req);
 
         assert req.txState() != null || fut.error() != null ||
@@ -1037,6 +1076,33 @@ else if (e instanceof IgniteTxHeuristicCheckedException) {
             (ctx.tm().tx(req.version()) == null && ctx.tm().nearTx(req.version()) == null);
     }
 
+    /**
+     * @param req Request.
+     */
+    private Collection<GridCacheContext> caches(GridDistributedTxPrepareRequest req) {
+        Set<GridCacheContext> caches = new HashSet<>();
+        
+        if (!F.isEmpty(req.reads())) {
+            for (IgniteTxEntry entry : req.reads()) {
+                GridCacheContext cctx = ctx.cacheContext(entry.cacheId());
+
+                if (cctx != null)
+                    caches.add(cctx);
+            }
+        }
+
+        if (!F.isEmpty(req.writes())) {
+            for (IgniteTxEntry entry : req.writes()) {
+                GridCacheContext cctx = ctx.cacheContext(entry.cacheId());
+
+                if (cctx != null)
+                    caches.add(cctx);
+            }
+        }
+
+        return caches;
+    }
+    
     /**
      * @param nodeId Node ID.
      * @param req Request.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
index df956b6e62972..ffd16fbe0f915 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridUpdateNotifier.java
@@ -17,16 +17,10 @@
 
 package org.apache.ignite.internal.processors.cluster;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.io.UnsupportedEncodingException;
-import java.net.URL;
-import java.net.URLConnection;
 import java.util.Collection;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractCacheTest.java
index 1f923f18ef4de..bab36a68d6a65 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractCacheTest.java
@@ -119,11 +119,14 @@ protected void putValid(String cacheName) {
      */
     protected void getInvalid(String cacheName) {
         try {
-            assert grid(0).cache(cacheName).get(KEY_VAL).equals(KEY_VAL);
-        }
-        catch (CacheException ignored) {
+            grid(0).cache(cacheName).get(KEY_VAL);
+
             assert false : "topology validation broken";
         }
+        catch (CacheException ex) {
+            assert ex.getCause() instanceof IgniteCheckedException &&
+                ex.getCause().getMessage().contains("cache topology is not valid");
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheTest.java
index fd386bbfbc023..95bac85704253 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheTest.java
@@ -59,8 +59,9 @@ public abstract class IgniteTopologyValidatorAbstractTxCacheTest extends IgniteT
         }
 
         assertEmpty(null); // rolled back
-        assertEmpty(CACHE_NAME_1); // rolled back
-        assertEmpty(CACHE_NAME_2); // rolled back
+
+        getInvalid(CACHE_NAME_1);
+        getInvalid(CACHE_NAME_2);
 
         try (Transaction tx = grid(0).transactions().txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
             putValid(null);
@@ -68,7 +69,7 @@ public abstract class IgniteTopologyValidatorAbstractTxCacheTest extends IgniteT
         }
 
         assertEmpty(null); // rolled back
-        assertEmpty(CACHE_NAME_1); // rolled back
+        getInvalid(CACHE_NAME_1); // rolled back
 
         startGrid(1);
 
@@ -94,7 +95,8 @@ public abstract class IgniteTopologyValidatorAbstractTxCacheTest extends IgniteT
         }
 
         assertEmpty(null); // rolled back
-        assertEmpty(CACHE_NAME_1); // rolled back
+
+        getInvalid(CACHE_NAME_1);
 
         try (Transaction tx = grid(0).transactions().txStart(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
             putValid(CACHE_NAME_1);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java
index 56eed411ce56b..99255e5d458d6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionRebalanceTest.java
@@ -51,7 +51,7 @@
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
 
 /**
- *
+ * Tests EVT_CACHE_REBALANCE_PART_DATA_LOST events.
  */
 public class GridLostPartitionRebalanceTest extends GridCommonAbstractTest {
     /** Cache name. */
@@ -120,12 +120,6 @@ public class GridLostPartitionRebalanceTest extends GridCommonAbstractTest {
         }
     };
 
-    /** {@inheritDoc} */
-    @Override protected void beforeTest() throws Exception {
-        latch = new CountDownLatch(expEvts);
-        cnt = new AtomicInteger(0);
-    }
-
     /**
      * @throws Exception If failed.
      */
@@ -150,6 +144,9 @@ public void testPartDataLostEventNoBackups() throws Exception {
      * @throws Exception If failed.
      */
     private void checkEvents() throws Exception {
+        latch = new CountDownLatch(expEvts);
+        cnt = new AtomicInteger(0);
+
         List<Ignite> srvrs = new ArrayList<>();
 
         // Client router. It always up, so client is guaranteed to get
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionValidationTest.java
new file mode 100644
index 0000000000000..db5ede8e2f29d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridLostPartitionValidationTest.java
@@ -0,0 +1,536 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.locks.Lock;
+import javax.cache.Cache;
+import javax.cache.CacheException;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.CacheRebalancingEvent;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.jetbrains.annotations.NotNull;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_VALIDATOR;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+
+/**
+ * Tests that cache operations validated.
+ */
+public class GridLostPartitionValidationTest extends GridCommonAbstractTest {
+    /** Cache name. */
+    private static final String CACHE_NAME = "test";
+
+    /** Tx cache name. */
+    private static final String TX_CACHE_NAME = "tx_test";
+
+    /** Backups. */
+    private int backups;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg.setBackups(backups);
+
+        CacheConfiguration ccfg2 = new CacheConfiguration(TX_CACHE_NAME);
+
+        ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg2.setBackups(backups);
+        ccfg2.setAtomicityMode(TRANSACTIONAL);
+
+        cfg.setCacheConfiguration(ccfg, ccfg2);
+
+        cfg.setIncludeEventTypes(EVT_CACHE_REBALANCE_PART_DATA_LOST);
+
+        Map<IgnitePredicate<? extends Event>, int[]> listeners = new HashMap<>();
+
+        listeners.put(new Listener(), new int[]{EVT_CACHE_REBALANCE_PART_DATA_LOST});
+
+        cfg.setLocalEventListeners(listeners);
+
+        cfg.setClientMode(gridName.contains("client"));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.clearProperty(IGNITE_CACHE_VALIDATOR);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty(IGNITE_CACHE_VALIDATOR, CacheValidator.class.getName());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartDataLostEvent1Backup() throws Exception {
+        backups = 1;
+
+        checkValidator();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    private void checkValidator() throws Exception {
+        List<Ignite> srvrs = new ArrayList<>();
+
+        srvrs.add(startGrid("server-0"));
+
+        final Ignite client = startGrid("client");
+
+        srvrs.add(startGrid("server-1"));
+        srvrs.add(startGrid("server-2"));
+
+        awaitPartitionMapExchange();
+
+        final IgniteCache<Object, Object> cache = client.cache(CACHE_NAME);
+        final IgniteCache<Object, Object> txCache = client.cache(TX_CACHE_NAME);
+
+        for (int i = 0; i < 10_000; i++) {
+            cache.put(i, i);
+            txCache.put(i, i);
+        }
+
+        // Stop node with 0 partition.
+        Set<ClusterNode> nodes = new HashSet<>(client.affinity(CACHE_NAME).mapPartitionToPrimaryAndBackups(0));
+
+        List<String> stopped = stopAffinityNodes(srvrs, nodes);
+
+        awaitPartitionMapExchange();
+
+        for (Iterator<Ignite> iter = srvrs.iterator(); iter.hasNext(); ) {
+            Ignite srvr = iter.next();
+
+            if (stopped.contains(srvr.name()))
+                iter.remove();
+        }
+
+        Ignite srvr = F.first(srvrs);
+
+        checkThrows(client);
+        checkThrows(srvr);
+    }
+
+    /**
+     * @param ignite Client.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    private void checkThrows(final Ignite ignite) {
+        final IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+        final IgniteCache<Object, Object> txCache = ignite.cache(TX_CACHE_NAME);
+
+        boolean client = ignite.cluster().localNode().isClient();
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                cache.query(new ScanQuery<>()).getAll();
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                cache.get(0);
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                cache.put(0, 0);
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                cache.remove(0);
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                cache.invoke(0, new CacheEntryProcessor<Object, Object, Object>() {
+                    @Override public Object process(MutableEntry<Object, Object> entry,
+                        Object... arguments) throws EntryProcessorException {
+
+                        return null;
+                    }
+                });
+
+                return null;
+            }
+        });
+
+        if (!client) {
+            assertThrows(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    cache.localPeek(0);
+
+                    return null;
+                }
+            });
+        }
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                Lock lock = cache.lock(0);
+
+                lock.lock();
+
+                return null;
+            }
+        });
+
+        if (!client) {
+            assertThrows(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    for (Cache.Entry<Object, Object> entry : cache.localEntries())
+                        System.out.println(entry);
+
+                    return null;
+                }
+            });
+        }
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                txCache.query(new ScanQuery<>()).getAll();
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                txCache.get(0);
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                txCache.put(0, 0);
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                txCache.remove(0);
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                txCache.invoke(0, new CacheEntryProcessor<Object, Object, Object>() {
+                    @Override public Object process(MutableEntry<Object, Object> entry,
+                        Object... arguments) throws EntryProcessorException {
+                        return null;
+                    }
+                });
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, READ_COMMITTED)) {
+                    txCache.get(0);
+
+                    tx.commit();
+                }
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, READ_COMMITTED)) {
+                    txCache.put(0, 0);
+
+                    tx.commit();
+                }
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, READ_COMMITTED)) {
+                    txCache.remove(0);
+
+                    tx.commit();
+                }
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, READ_COMMITTED)) {
+                    txCache.invoke(0, new CacheEntryProcessor<Object, Object, Object>() {
+                        @Override public Object process(MutableEntry<Object, Object> entry,
+                            Object... arguments) throws EntryProcessorException {
+                            return null;
+                        }
+                    });
+
+                    tx.commit();
+                }
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) {
+                    txCache.get(0);
+
+                    tx.commit();
+                }
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) {
+                    txCache.put(0, 0);
+
+                    tx.commit();
+                }
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) {
+                    txCache.remove(0);
+
+                    tx.commit();
+                }
+
+                return null;
+            }
+        });
+
+        assertThrows(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) {
+                    txCache.invoke(0, new CacheEntryProcessor<Object, Object, Object>() {
+                        @Override public Object process(MutableEntry<Object, Object> entry,
+                            Object... arguments) throws EntryProcessorException {
+
+                            return null;
+                        }
+                    });
+
+                    tx.commit();
+                }
+
+                return null;
+            }
+        });
+
+        if (!client) {
+            assertThrows(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    txCache.localPeek(0);
+
+                    return null;
+                }
+            });
+        }
+
+        if (!client) {
+            assertThrows(new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    for (Cache.Entry<Object, Object> entry : txCache.localEntries())
+                        System.out.println(entry);
+
+                    return null;
+                }
+            });
+        }
+    }
+
+    /**
+     * @param c Closure.
+     */
+    private void assertThrows(Callable<?> c) {
+        try {
+            c.call();
+
+            assert false : "Exception was not thrown";
+        }
+        catch (CacheException | IgniteException e) {
+            log.info("Caught expected exception: " + e.getClass());
+        }
+        catch (Exception e) {
+            e.printStackTrace();
+
+            assert false : "Wrong exception was thrown: " + e.getClass();
+        }
+    }
+
+    /**
+     * @param srvrs Servers.
+     * @param nodes Nodes.
+     */
+    @NotNull private List<String> stopAffinityNodes(List<Ignite> srvrs, Set<ClusterNode> nodes) throws IgniteCheckedException {
+        List<IgniteInternalFuture<?>> futs = new ArrayList<>();
+
+        final List<String> stopped = new ArrayList<>();
+
+        for (final Ignite srv : srvrs) {
+            final ClusterNode node = srv.cluster().localNode();
+
+            if (nodes.contains(node)) {
+                IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+                    @Override public Object call() throws Exception {
+                        srv.close();
+
+                        System.out.println(">> Stopped " + srv.name() + " " + node.id());
+
+                        stopped.add(srv.name());
+
+                        return null;
+                    }
+                });
+
+                futs.add(fut);
+            }
+        }
+
+        for (IgniteInternalFuture<?> fut : futs)
+            fut.get();
+
+        return stopped;
+    }
+
+    /**
+     *
+     */
+    private static class Listener implements IgnitePredicate<CacheRebalancingEvent> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** Ignite. */
+        @SuppressWarnings("unused")
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(CacheRebalancingEvent evt) {
+            ignite.cluster().nodeLocalMap().putIfAbsent(evt.cacheName(), false);
+
+            return true;
+        }
+    }
+
+    /**
+     *
+     */
+    public static class CacheValidator implements IgniteClosure<String, Throwable> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Logger. */
+        @SuppressWarnings("unused")
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** {@inheritDoc} */
+        @Override public Throwable apply(String cacheName) {
+            log.info(">>> Validator");
+
+            Object val = ignite.cluster().nodeLocalMap().get(cacheName);
+
+            return Boolean.FALSE.equals(val) ? new IllegalStateException("Illegal cache state "
+                + Thread.currentThread().getName()) : null;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index 7a66883f93f24..6ae7d22d1b138 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -60,6 +60,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridLostPartitionRebalanceTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridLostPartitionValidationTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheDaemonNodeReplicatedSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicGetAndTransformStoreSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedBasicApiTest;
@@ -155,6 +156,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
         suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
         suite.addTestSuite(GridLostPartitionRebalanceTest.class);
+        suite.addTestSuite(GridLostPartitionValidationTest.class);
 
         // Test for byte array value special case.
         suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 85fb4989658d3..4080c1ff06bfc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -48,6 +48,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -85,7 +86,6 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
 import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
-import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
 import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
@@ -871,6 +871,9 @@ public GridQueryFieldsResult queryLocalSqlFields(@Nullable final String spaceNam
 
         Prepared p = GridSqlQueryParser.prepared((JdbcPreparedStatement)stmt);
 
+        if (ctx != null && ctx.cache().cacheValidator() != null)
+            validateCaches(p);
+
         if (!p.isQuery()) {
             SqlFieldsQuery fldsQry = new SqlFieldsQuery(qry);
 
@@ -1287,7 +1290,15 @@ public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable
         runs.put(run.id(), run);
 
         try {
-            ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, cancel);
+            PreparedStatement stmt = preparedStatementWithParams(conn, sql, params, true);
+
+            if (ctx != null && ctx.cache().cacheValidator() != null) {
+                Prepared p = GridSqlQueryParser.prepared((JdbcPreparedStatement)stmt);
+
+                validateCaches(p);
+            }
+
+            ResultSet rs = executeSqlQueryWithTimer(spaceName, stmt, conn, sql, params, 0, cancel);
 
             return new KeyValIterator(rs);
         }
@@ -1633,7 +1644,7 @@ else if (star > 0) {
                 (upper.startsWith("WHERE") || upper.startsWith("ORDER") || upper.startsWith("LIMIT") ?
                     " " : " WHERE ");
 
-        if(tableAlias != null)
+        if (tableAlias != null)
             t = tableAlias;
 
         qry = "SELECT " + t + "." + KEY_FIELD_NAME + ", " + t + "." + VAL_FIELD_NAME + from + qry;
@@ -2928,6 +2939,26 @@ public FieldsIterator(ResultSet data) throws IgniteCheckedException {
         }
     }
 
+    /**
+     * @param p Prepared.
+     */
+    private void validateCaches(Prepared p) throws IgniteCheckedException {
+        Set<GridH2Table> tbls = GridSqlQueryParser.getTables(p);
+
+        if (!F.isEmpty(tbls)) {
+            for (GridH2Table tbl : tbls) {
+                if (tbl != null) {
+                    GridCacheContext<?, ?> cctx = tbl.rowDescriptor().context();
+
+                    Throwable exc = cctx.topologyVersionFuture().validateCache(cctx);
+
+                    if (exc != null)
+                        throw new IgniteCheckedException(exc);
+                }
+            }
+        }
+    }
+
     /**
      * Special key/value iterator based on database result set.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index 0f940e9c3846c..e13b61833bdb4 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -22,12 +22,15 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
 import org.h2.command.Command;
 import org.h2.command.CommandContainer;
 import org.h2.command.Prepared;
@@ -1062,6 +1065,48 @@ private GridSqlElement parseExpression0(Expression expression, boolean calcTypes
             expression.getClass().getSimpleName() + ']');
     }
 
+    /**
+     * @param stmt Prepared statement.
+     * @return Tables set.
+     */
+    public static Set<GridH2Table> getTables(Prepared stmt) {
+        Set<GridH2Table> res = new HashSet<>();
+
+        Set<Table> tbls = null;
+
+        if (stmt instanceof Query)
+            tbls = ((Query)stmt).getTables();
+
+        else if (stmt instanceof Merge)
+            tbls = Collections.singleton(MERGE_TABLE.get((Merge)stmt));
+
+        else if (stmt instanceof Insert)
+            tbls = Collections.singleton(INSERT_TABLE.get((Insert)stmt));
+
+        else if (stmt instanceof Delete) {
+            TableFilter filter = DELETE_FROM.get((Delete)stmt);
+
+            if (filter != null)
+                tbls = Collections.singleton(filter.getTable());
+        }
+
+        else if (stmt instanceof Update) {
+            TableFilter filter = UPDATE_TARGET.get((Update)stmt);
+
+            if (filter != null)
+                tbls = Collections.singleton(filter.getTable());
+        }
+
+        if (tbls != null) {
+            for (Table table : tbls) {
+                if (table instanceof GridH2Table)
+                    res.add((GridH2Table)table);
+            }
+        }
+
+        return res;
+    }
+
     /**
      * @param cond Condition.
      * @param o Object.
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index fc002cb3ce200..bac7c8c16e6b7 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -554,7 +554,7 @@ private void onQueryRequest0(
                 }
             }
 
-            qr = new QueryResults(reqId, qrys.size(), mainCctx);
+            qr = new QueryResults(reqId, qrys.size(), mainCctx, cacheIds);
 
             if (nodeRess.put(reqId, segmentId, qr) != null)
                 throw new IllegalStateException();
@@ -649,6 +649,12 @@ private void onQueryRequest0(
                         throw new QueryCancelledException();
                     }
 
+                    // Validate cache
+                    Throwable exc = qr.validateCaches();
+
+                    if (exc != null)
+                        throw exc;
+
                     // Send the first page.
                     sendNextPage(nodeRess, node, qr, qryIdx, segmentId, pageSize);
 
@@ -741,8 +747,17 @@ private void onNextPageRequest(ClusterNode node, GridQueryNextPageRequest req) {
             sendError(node, req.queryRequestId(), new CacheException("No query result found for request: " + req));
         else if (qr.canceled)
             sendError(node, req.queryRequestId(), new QueryCancelledException());
-        else
-            sendNextPage(nodeRess, node, qr, req.query(), req.segmentId(), req.pageSize());
+        else {
+            Throwable exc = qr.validateCaches();
+
+            if (exc == null)
+                sendNextPage(nodeRess, node, qr, req.query(), req.segmentId(), req.pageSize());
+            else {
+                nodeRess.remove(req.queryRequestId(), req.segmentId(), qr);
+
+                sendError(node, req.queryRequestId(), new CacheException(exc.getMessage(), exc));
+            }
+        }
     }
 
     /**
@@ -970,6 +985,9 @@ private class QueryResults {
         /** */
         private final GridCacheContext<?,?> cctx;
 
+        /** */
+        private final List<Integer> cacheIds;
+
         /** */
         private volatile boolean canceled;
 
@@ -977,11 +995,14 @@ private class QueryResults {
          * @param qryReqId Query request ID.
          * @param qrys Number of queries.
          * @param cctx Cache context.
+         * @param cacheIds Cache IDs.
          */
         @SuppressWarnings("unchecked")
-        private QueryResults(long qryReqId, int qrys, GridCacheContext<?, ?> cctx) {
+        private QueryResults(long qryReqId, int qrys, GridCacheContext<?, ?> cctx,
+            List<Integer> cacheIds) {
             this.qryReqId = qryReqId;
             this.cctx = cctx;
+            this.cacheIds = cacheIds;
 
             results = new AtomicReferenceArray<>(qrys);
             cancels = new GridQueryCancel[qrys];
@@ -1049,6 +1070,27 @@ void cancel(boolean forceQryCancel) {
                 }
             }
         }
+
+        /**
+         * Validate caches.
+         *
+         * @return {@link Throwable} if cache is not valid or {@code null} otherwise.
+         */
+        Throwable validateCaches() {
+            // Validate cache
+            for (Integer cacheId : cacheIds) {
+                GridCacheContext cctx = ctx.cache().context().cacheContext(cacheId);
+
+                if (!cctx.isLocal()) {
+                    Throwable exc = cctx.topologyVersionFuture().validateCache(cctx);
+
+                    if (exc != null)
+                        return exc;
+                }
+            }
+
+            return null;
+        }
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridLostPartitionValidationQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridLostPartitionValidationQueryTest.java
new file mode 100644
index 0000000000000..5ca7938a2491c
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridLostPartitionValidationQueryTest.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.CacheRebalancingEvent;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_VALIDATOR;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
+
+/**
+ * Tests that SQL query validated.
+ */
+public class GridLostPartitionValidationQueryTest extends GridCommonAbstractTest {
+    /** Cache name. */
+    private static final String CACHE_NAME = "test";
+
+    /** Tx cache name. */
+    private static final String TX_CACHE_NAME = "tx_test";
+
+    /** Backups. */
+    private int backups;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg1 = new CacheConfiguration(CACHE_NAME);
+
+        ccfg1.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg1.setBackups(backups);
+
+        QueryEntity qe1 = new QueryEntity(Integer.class.getName(), Integer.class.getName());
+        qe1.setTableName("T1");
+
+        ccfg1.setQueryEntities(Collections.singleton(qe1));
+
+        CacheConfiguration ccfg2 = new CacheConfiguration(TX_CACHE_NAME);
+
+        ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg2.setBackups(backups);
+        ccfg2.setAtomicityMode(TRANSACTIONAL);
+
+        QueryEntity qe2 = new QueryEntity(Integer.class.getName(), Integer.class.getName());
+        qe2.setTableName("T2");
+
+        ccfg2.setQueryEntities(Collections.singleton(qe2));
+
+        cfg.setCacheConfiguration(ccfg1, ccfg2);
+
+        cfg.setIncludeEventTypes(EVT_CACHE_REBALANCE_PART_DATA_LOST);
+
+        Map<IgnitePredicate<? extends Event>, int[]> listeners = new HashMap<>();
+
+        listeners.put(new Listener(), new int[]{EVT_CACHE_REBALANCE_PART_DATA_LOST});
+
+        cfg.setLocalEventListeners(listeners);
+
+        cfg.setClientMode(gridName.contains("client"));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.clearProperty(IGNITE_CACHE_VALIDATOR);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty(IGNITE_CACHE_VALIDATOR, CacheValidator.class.getName());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartDataLostEvent1Backup() throws Exception {
+        backups = 1;
+
+        checkValidator();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    private void checkValidator() throws Exception {
+        List<Ignite> srvrs = new ArrayList<>();
+
+        srvrs.add(startGrid("server-0"));
+
+        final Ignite client = startGrid("client");
+
+        srvrs.add(startGrid("server-1"));
+        srvrs.add(startGrid("server-2"));
+
+        awaitPartitionMapExchange();
+
+        final IgniteCache<Object, Object> cache = client.cache(CACHE_NAME);
+
+        IgniteDataStreamer<Object, Object> streamer1 = client.dataStreamer(CACHE_NAME);
+        IgniteDataStreamer<Object, Object> streamer2 = client.dataStreamer(TX_CACHE_NAME);
+
+        for (int i = 0; i < 100_000; i++) {
+            streamer1.addData(i, i);
+            streamer2.addData(i, i);
+        }
+
+        streamer1.close();
+        streamer2.close();
+
+        @SuppressWarnings("unchecked")
+        final SqlQuery qry1 = new SqlQuery(Integer.class, "SELECT a.* FROM T1 a, \"tx_test\".T2 b LIMIT 10000");
+        final SqlFieldsQuery qry2 = new SqlFieldsQuery("SELECT a.* FROM T1 a, \"tx_test\".T2 b LIMIT 10000");
+
+        // Stop node with 0 partition.
+        Set<ClusterNode> nodes = new HashSet<>(client.affinity(CACHE_NAME).mapPartitionToPrimaryAndBackups(0));
+
+        List<String> stopped = stopAffinityNodes(srvrs, nodes);
+
+        awaitPartitionMapExchange();
+
+        for (Iterator<Ignite> iter = srvrs.iterator(); iter.hasNext(); ) {
+            Ignite srvr = iter.next();
+
+            if (stopped.contains(srvr.name()))
+                iter.remove();
+        }
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                cache.query(qry1).getAll();
+
+                return null;
+            }
+        }, CacheException.class, null);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                cache.query(qry2).getAll();
+
+                return null;
+            }
+        }, CacheException.class, null);
+
+        final IgniteCache<Object, Object> srvCache = srvrs.get(0).cache(CACHE_NAME);
+
+        qry1.setLocal(true);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                srvCache.query(qry1).getAll();
+
+                return null;
+            }
+        }, CacheException.class, null);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                srvCache.query(qry2).getAll();
+
+                return null;
+            }
+        }, CacheException.class, null);
+    }
+
+    /**
+     * @param srvrs Servers.
+     * @param nodes Nodes.
+     */
+    @NotNull private List<String> stopAffinityNodes(List<Ignite> srvrs, Set<ClusterNode> nodes) throws IgniteCheckedException {
+        List<IgniteInternalFuture<?>> futs = new ArrayList<>();
+
+        final List<String> stopped = new ArrayList<>();
+
+        for (final Ignite srv : srvrs) {
+            final ClusterNode node = srv.cluster().localNode();
+
+            if (nodes.contains(node)) {
+                IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+                    @Override public Object call() throws Exception {
+                        srv.close();
+
+                        System.out.println(">> Stopped " + srv.name() + " " + node.id());
+
+                        stopped.add(srv.name());
+
+                        return null;
+                    }
+                });
+
+                futs.add(fut);
+            }
+        }
+
+        for (IgniteInternalFuture<?> fut : futs)
+            fut.get();
+
+        return stopped;
+    }
+
+    /**
+     *
+     */
+    private static class Listener implements IgnitePredicate<CacheRebalancingEvent> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** Ignite. */
+        @SuppressWarnings("unused")
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(CacheRebalancingEvent evt) {
+            if (TX_CACHE_NAME.equals(evt.cacheName()))
+                ignite.cluster().nodeLocalMap().putIfAbsent(evt.cacheName(), false);
+
+            return true;
+        }
+    }
+
+    /**
+     *
+     */
+    public static class CacheValidator implements IgniteClosure<String, Throwable> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Logger. */
+        @SuppressWarnings("unused")
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** {@inheritDoc} */
+        @Override public Throwable apply(String cacheName) {
+            log.info(">>> Validator " + cacheName);
+
+            Object val = ignite.cluster().nodeLocalMap().get(cacheName);
+
+            return Boolean.FALSE.equals(val) ? new IllegalStateException("Illegal cache state "
+                + Thread.currentThread().getName()) : null;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index 79492149d753e..ceeb022e83977 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheOffheapIndexGetSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheSwapSelfTest;
 import org.apache.ignite.internal.processors.cache.GridIndexingWithNoopSwapSelfTest;
+import org.apache.ignite.internal.processors.cache.GridLostPartitionValidationQueryTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheConfigurationPrimitiveTypesSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheStarvationOnRebalanceTest;
 import org.apache.ignite.internal.processors.cache.IgniteClientReconnectQueriesTest;
@@ -84,6 +85,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(ClientReconnectAfterClusterRestartTest.class);
 
+        suite.addTestSuite(GridLostPartitionValidationQueryTest.class);
+
         return suite;
     }
 }

From 31944326c9aba6dac08482c5cadcd562ba7577d1 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Fri, 27 Oct 2017 17:37:58 +0300
Subject: [PATCH 409/516] revert previous changes on IGNITE-5860

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  52 ++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 250 +++++++++---------
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |   2 +-
 .../tcp/TcpClientDiscoverySpiSelfTest.java    | 229 +---------------
 4 files changed, 149 insertions(+), 384 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index ce4f1b7fe4b9a..c32c8e057d9fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -26,7 +26,6 @@
 import java.net.SocketTimeoutException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -468,8 +467,7 @@ else if (state == DISCONNECTED) {
     }
 
     /**
-     * @param prevAddr If reconnect is in progress, then previous address of the router the client was connected to
-     * and {@code null} otherwise.
+     * @param recon {@code True} if reconnects.
      * @param timeout Timeout.
      * @return Opened socket or {@code null} if timeout.
      * @throws InterruptedException If interrupted.
@@ -477,9 +475,9 @@ else if (state == DISCONNECTED) {
      * @see TcpDiscoverySpi#joinTimeout
      */
     @SuppressWarnings("BusyWait")
-    @Nullable private T2<SocketStream, Boolean> joinTopology(InetSocketAddress prevAddr, long timeout)
+    @Nullable private T2<SocketStream, Boolean> joinTopology(boolean recon, long timeout)
         throws IgniteSpiException, InterruptedException {
-        List<InetSocketAddress> addrs = null;
+        Collection<InetSocketAddress> addrs = null;
 
         long startTime = U.currentTimeMillis();
 
@@ -488,7 +486,7 @@ else if (state == DISCONNECTED) {
                 throw new InterruptedException();
 
             while (addrs == null || addrs.isEmpty()) {
-                addrs = new ArrayList<>(spi.resolvedAddresses());
+                addrs = spi.resolvedAddresses();
 
                 if (!F.isEmpty(addrs)) {
                     if (log.isDebugEnabled())
@@ -508,25 +506,17 @@ else if (state == DISCONNECTED) {
                 }
             }
 
-            // process failed node last
-            if (prevAddr != null) {
-                int idx = addrs.indexOf(prevAddr);
-
-                if (idx != -1)
-                    Collections.swap(addrs, idx, 0);
-            }
-
             Collection<InetSocketAddress> addrs0 = new ArrayList<>(addrs);
 
+            Iterator<InetSocketAddress> it = addrs.iterator();
+
             boolean wait = false;
 
-            for (int i = addrs.size() - 1; i >= 0; i--) {
+            while (it.hasNext()) {
                 if (Thread.currentThread().isInterrupted())
                     throw new InterruptedException();
 
-                InetSocketAddress addr = addrs.get(i);
-
-                boolean recon = prevAddr != null;
+                InetSocketAddress addr = it.next();
 
                 T3<SocketStream, Integer, Boolean> sockAndRes;
 
@@ -540,7 +530,7 @@ else if (state == DISCONNECTED) {
                 }
 
                 if (sockAndRes == null) {
-                    addrs.remove(i);
+                    it.remove();
 
                     continue;
                 }
@@ -862,8 +852,8 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
     }
 
     /** {@inheritDoc} */
-    @Override protected Collection<IgniteSpiThread> threads() {
-        return Arrays.asList(sockWriter, msgWorker);
+    @Override protected IgniteSpiThread workerThread() {
+        return msgWorker;
     }
 
     /**
@@ -1349,20 +1339,15 @@ private class Reconnector extends IgniteSpiThread {
         private boolean clientAck;
 
         /** */
-        private final boolean join;
-
-        /** */
-        private final InetSocketAddress prevAddr;
+        private boolean join;
 
         /**
          * @param join {@code True} if reconnects during join.
-         * @param prevAddr Address of the node, that this client was previously connected to.
          */
-        protected Reconnector(boolean join, InetSocketAddress prevAddr) {
+        protected Reconnector(boolean join) {
             super(spi.ignite().name(), "tcp-client-disco-reconnector", log);
 
             this.join = join;
-            this.prevAddr = prevAddr;
         }
 
         /**
@@ -1392,7 +1377,7 @@ public void cancel() {
 
             try {
                 while (true) {
-                    T2<SocketStream, Boolean> joinRes = joinTopology(prevAddr, timeout);
+                    T2<SocketStream, Boolean> joinRes = joinTopology(true, timeout);
 
                     if (joinRes == null) {
                         if (join) {
@@ -1627,10 +1612,6 @@ else if (msg instanceof TcpDiscoveryNodeFailedMessage &&
                     }
                     else if (msg instanceof SocketClosedMessage) {
                         if (((SocketClosedMessage)msg).sock == currSock) {
-                            Socket sock = currSock.sock;
-
-                            InetSocketAddress prevAddr = new InetSocketAddress(sock.getInetAddress(), sock.getPort());
-
                             currSock = null;
 
                             boolean join = joinLatch.getCount() > 0;
@@ -1659,7 +1640,8 @@ else if (msg instanceof SocketClosedMessage) {
 
                                     assert reconnector == null;
 
-                                    reconnector = new Reconnector(join, prevAddr);
+                                    final Reconnector reconnector = new Reconnector(join);
+                                    this.reconnector = reconnector;
                                     reconnector.start();
                                 }
                             }
@@ -1830,7 +1812,7 @@ private void tryJoin() throws InterruptedException {
 
             T2<SocketStream, Boolean> joinRes;
             try {
-                joinRes = joinTopology(null, spi.joinTimeout);
+                joinRes = joinTopology(false, spi.joinTimeout);
             }
             catch (IgniteSpiException e) {
                 joinError(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 640dcec85b0b0..189b37a6a0899 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -222,9 +222,6 @@ class ServerImpl extends TcpDiscoveryImpl {
     /** Pending custom messages that should not be sent between NodeAdded and NodeAddFinished messages. */
     private Queue<TcpDiscoveryCustomEventMessage> pendingCustomMsgs = new ArrayDeque<>();
 
-    /** Messages history used for client reconnect. */
-    private final EnsuredMessageHistory msgHist = new EnsuredMessageHistory();
-
     /** If non-shared IP finder is used this flag shows whether IP finder contains local address. */
     private boolean ipFinderHasLocAddr;
 
@@ -1645,23 +1642,8 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
     }
 
     /** {@inheritDoc} */
-    @Override protected Collection<IgniteSpiThread> threads() {
-        Collection<IgniteSpiThread> threads;
-
-        synchronized (mux) {
-            threads = new ArrayList<>(readers.size() + clientMsgWorkers.size() + 4);
-            threads.addAll(readers);
-        }
-
-        threads.addAll(clientMsgWorkers.values());
-        threads.add(tcpSrvr);
-        threads.add(ipFinderCleaner);
-        threads.add(msgWorker);
-        threads.add(statsPrinter);
-
-        threads.removeAll(Collections.<IgniteSpiThread>singleton(null));
-
-        return threads;
+    @Override protected IgniteSpiThread workerThread() {
+        return msgWorker;
     }
 
     /**
@@ -2109,9 +2091,7 @@ else if (msg instanceof TcpDiscoveryNodeLeftMessage)
             else if (msg instanceof TcpDiscoveryNodeFailedMessage)
                 clearClientAddFinished(((TcpDiscoveryNodeFailedMessage)msg).failedNodeId());
 
-            synchronized (msgs) {
-                msgs.add(msg);
-            }
+            msgs.add(msg);
         }
 
         /**
@@ -2173,16 +2153,14 @@ private boolean mapsEqual(Map<Integer, byte[]> m1, Map<Integer, byte[]> m2) {
                 // Client connection failed before it received TcpDiscoveryNodeAddedMessage.
                 List<TcpDiscoveryAbstractMessage> res = null;
 
-                synchronized (msgs) {
-                    for (TcpDiscoveryAbstractMessage msg : msgs) {
-                        if (msg instanceof TcpDiscoveryNodeAddedMessage) {
-                            if (node.id().equals(((TcpDiscoveryNodeAddedMessage)msg).node().id()))
-                                res = new ArrayList<>(msgs.size());
-                        }
-
-                        if (res != null)
-                            res.add(prepare(msg, node.id()));
+                for (TcpDiscoveryAbstractMessage msg : msgs) {
+                    if (msg instanceof TcpDiscoveryNodeAddedMessage) {
+                        if (node.id().equals(((TcpDiscoveryNodeAddedMessage)msg).node().id()))
+                            res = new ArrayList<>(msgs.size());
                     }
+
+                    if (res != null)
+                        res.add(prepare(msg, node.id()));
                 }
 
                 if (log.isDebugEnabled()) {
@@ -2195,26 +2173,20 @@ private boolean mapsEqual(Map<Integer, byte[]> m1, Map<Integer, byte[]> m2) {
                 return res;
             }
             else {
-                Collection<TcpDiscoveryAbstractMessage> cp;
+                if (msgs.isEmpty())
+                    return Collections.emptyList();
 
-                boolean skip;
+                Collection<TcpDiscoveryAbstractMessage> cp = new ArrayList<>(msgs.size());
 
-                synchronized (msgs) {
-                    if (msgs.isEmpty())
-                        return Collections.emptyList();
+                boolean skip = true;
 
-                    cp = new ArrayList<>(msgs.size());
-
-                    skip = true;
-
-                    for (TcpDiscoveryAbstractMessage msg : msgs) {
-                        if (skip) {
-                            if (msg.id().equals(lastMsgId))
-                                skip = false;
-                        }
-                        else
-                            cp.add(prepare(msg, node.id()));
+                for (TcpDiscoveryAbstractMessage msg : msgs) {
+                    if (skip) {
+                        if (msg.id().equals(lastMsgId))
+                            skip = false;
                     }
+                    else
+                        cp.add(prepare(msg, node.id()));
                 }
 
                 cp = !skip ? cp : null;
@@ -2503,6 +2475,9 @@ private class RingMessageWorker extends MessageWorkerAdapter<TcpDiscoveryAbstrac
         /** Pending messages. */
         private final PendingMessages pendingMsgs = new PendingMessages();
 
+        /** Messages history used for client reconnect. */
+        private final EnsuredMessageHistory msgHist = new EnsuredMessageHistory();
+
         /** Last message that updated topology. */
         private TcpDiscoveryAbstractMessage lastMsg;
 
@@ -2710,6 +2685,9 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage)
             else
                 assert false : "Unknown message type: " + msg.getClass().getSimpleName();
 
+            if (ensured && redirectToClients(msg))
+                msgHist.add(msg);
+
             if (msg.senderNodeId() != null && !msg.senderNodeId().equals(getLocalNodeId())) {
                 // Received a message from remote node.
                 onMessageExchanged();
@@ -2742,9 +2720,6 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage)
          */
         private void sendMessageToClients(TcpDiscoveryAbstractMessage msg) {
             if (redirectToClients(msg)) {
-                if (spi.ensured(msg))
-                    msgHist.add(msg);
-
                 byte[] msgBytes = null;
 
                 for (ClientMessageWorker clientMsgWorker : clientMsgWorkers.values()) {
@@ -3896,6 +3871,9 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                 nodeAddedMsg.client(msg.client());
 
                 processNodeAddedMessage(nodeAddedMsg);
+
+                if (nodeAddedMsg.verified())
+                    msgHist.add(nodeAddedMsg);
             }
             else if (sendMessageToRemotes(msg))
                 sendMessageAcrossRing(msg);
@@ -3984,34 +3962,94 @@ private void trySendMessageDirectly(TcpDiscoveryNode node, TcpDiscoveryAbstractM
         }
 
         /**
-         * Forward client reconnect message to coordinator.
+         * Processes client reconnect message.
          *
          * @param msg Client reconnect message.
          */
         private void processClientReconnectMessage(TcpDiscoveryClientReconnectMessage msg) {
             UUID nodeId = msg.creatorNodeId();
+
             UUID locNodeId = getLocalNodeId();
 
-            TcpDiscoveryNode node = ring.node(nodeId);
+            boolean isLocNodeRouter = locNodeId.equals(msg.routerNodeId());
 
-            assert node == null || node.isClient();
+            if (!msg.verified()) {
+                TcpDiscoveryNode node = ring.node(nodeId);
 
-            if (node != null) {
-                node.clientRouterNodeId(msg.routerNodeId());
-                node.aliveCheck(spi.maxMissedClientHbs);
-            }
+                assert node == null || node.isClient();
 
-            if (!isLocalNodeCoordinator() && sendMessageToRemotes(msg))
-                sendMessageAcrossRing(msg);
+                if (node != null) {
+                    node.clientRouterNodeId(msg.routerNodeId());
+                    node.aliveCheck(spi.maxMissedClientHbs);
+                }
+
+                if (isLocalNodeCoordinator()) {
+                    msg.verify(locNodeId);
+
+                    if (node != null) {
+                        Collection<TcpDiscoveryAbstractMessage> pending = msgHist.messages(msg.lastMessageId(), node);
+
+                        if (pending != null) {
+                            msg.pendingMessages(pending);
+                            msg.success(true);
+
+                            if (log.isDebugEnabled())
+                                log.debug("Accept client reconnect, restored pending messages " +
+                                    "[locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
+                        }
+                        else {
+                            if (log.isDebugEnabled())
+                                log.debug("Failing reconnecting client node because failed to restore pending " +
+                                    "messages [locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
+
+                            TcpDiscoveryNodeFailedMessage nodeFailedMsg = new TcpDiscoveryNodeFailedMessage(locNodeId,
+                                node.id(), node.internalOrder());
+
+                            processNodeFailedMessage(nodeFailedMsg);
+
+                            if (nodeFailedMsg.verified())
+                                msgHist.add(nodeFailedMsg);
+                        }
+                    }
+                    else if (log.isDebugEnabled())
+                        log.debug("Reconnecting client node is already failed [nodeId=" + nodeId + ']');
+
+                    if (isLocNodeRouter) {
+                        ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
+
+                        if (wrk != null)
+                            wrk.addMessage(msg);
+                        else if (log.isDebugEnabled())
+                            log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
+                                locNodeId + ", clientNodeId=" + nodeId + ']');
+                    }
+                    else {
+                        if (sendMessageToRemotes(msg))
+                            sendMessageAcrossRing(msg);
+                    }
+                }
+                else {
+                    if (sendMessageToRemotes(msg))
+                        sendMessageAcrossRing(msg);
+                }
+            }
+            else {
+                if (isLocalNodeCoordinator())
+                    addMessage(new TcpDiscoveryDiscardMessage(locNodeId, msg.id(), false));
 
-            if (msg.verified() && msg.routerNodeId().equals(getLocalNodeId())) {
-                ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
+                if (isLocNodeRouter) {
+                    ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
 
-                if (wrk != null)
-                    wrk.addMessage(msg);
-                else if (log.isDebugEnabled())
-                    log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
-                        locNodeId + ", clientNodeId=" + nodeId + ']');
+                    if (wrk != null)
+                        wrk.addMessage(msg);
+                    else if (log.isDebugEnabled())
+                        log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
+                            locNodeId + ", clientNodeId=" + nodeId + ']');
+                }
+                else {
+                    if (ring.hasRemoteNodes() && !isLocalNodeCoordinator())
+                        sendMessageAcrossRing(msg);
+                }
             }
         }
 
@@ -4055,6 +4093,9 @@ private void processNodeAddedMessage(TcpDiscoveryNodeAddedMessage msg) {
 
                     processNodeAddFinishedMessage(addFinishMsg);
 
+                    if (addFinishMsg.verified())
+                        msgHist.add(addFinishMsg);
+
                     addMessage(new TcpDiscoveryDiscardMessage(locNodeId, msg.id(), false));
 
                     return;
@@ -5115,6 +5156,9 @@ private void processHeartbeatMessage(TcpDiscoveryHeartbeatMessage msg) {
                                             locNodeId, clientNode.id(), clientNode.internalOrder());
 
                                         processNodeFailedMessage(nodeFailedMsg);
+
+                                        if (nodeFailedMsg.verified())
+                                            msgHist.add(nodeFailedMsg);
                                     }
                                 }
                             }
@@ -5312,6 +5356,9 @@ private void processCustomMessage(TcpDiscoveryCustomEventMessage msg) {
                                 ackMsg.topologyVersion(msg.topologyVersion());
 
                                 processCustomMessage(ackMsg);
+
+                                if (ackMsg.verified())
+                                    msgHist.add(ackMsg);
                             }
                             catch (IgniteCheckedException e) {
                                 U.error(log, "Failed to marshal discovery custom message.", e);
@@ -5413,8 +5460,12 @@ private void checkPendingCustomMessages() {
             if (joiningEmpty && isLocalNodeCoordinator()) {
                 TcpDiscoveryCustomEventMessage msg;
 
-                while ((msg = pollPendingCustomeMessage()) != null)
+                while ((msg = pollPendingCustomeMessage()) != null) {
                     processCustomMessage(msg);
+
+                    if (msg.verified())
+                        msgHist.add(msg);
+                }
             }
         }
 
@@ -5968,7 +6019,7 @@ else if (msg instanceof TcpDiscoveryClientReconnectMessage) {
                                     if (clientMsgWrk.getState() == State.NEW)
                                         clientMsgWrk.start();
 
-                                    processClientReconnectMessage((TcpDiscoveryClientReconnectMessage)msg);
+                                    msgWorker.addMessage(msg);
 
                                     continue;
                                 }
@@ -6205,67 +6256,6 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage) {
             }
         }
 
-        /**
-         * Processes client reconnect message.
-         *
-         * @param msg Client reconnect message.
-         */
-        private void processClientReconnectMessage(TcpDiscoveryClientReconnectMessage msg) {
-            UUID nodeId = msg.creatorNodeId();
-            UUID locNodeId = getLocalNodeId();
-
-            boolean isLocNodeRouter = locNodeId.equals(msg.routerNodeId());
-
-            if (isLocNodeRouter) {
-                TcpDiscoveryNode node = ring.node(nodeId);
-                ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
-
-                if (wrk != null && node != null) {
-                    if (!msg.verified()) {
-                        msg.verify(getLocalNodeId());
-
-                        Collection<TcpDiscoveryAbstractMessage> pending = msgHist.messages(msg.lastMessageId(), node);
-
-                        if (pending != null) {
-                            msg.success(true);
-                            msg.pendingMessages(pending);
-
-                            TcpDiscoveryClientReconnectMessage msgCp = new TcpDiscoveryClientReconnectMessage(
-                                msg.creatorNodeId(), msg.routerNodeId(), msg.lastMessageId());
-                            msgCp.client(msg.client());
-
-                            msgWorker.addMessage(msgCp);
-
-                            if (log.isDebugEnabled()) {
-                                log.debug("Accept client reconnect, restored pending messages " +
-                                    "[locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
-                            }
-                        }
-                        else {
-                            if (log.isDebugEnabled())
-                                log.debug("Failing reconnecting client node because failed to restore pending " +
-                                    "messages [locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
-
-                            TcpDiscoveryNodeFailedMessage nodeFailedMsg = new TcpDiscoveryNodeFailedMessage(locNodeId,
-                                node.id(), node.internalOrder());
-
-                            msgWorker.addMessage(nodeFailedMsg);
-                        }
-                    }
-                    else
-                        wrk.addMessage(msg);
-                }
-                else if (log.isDebugEnabled())
-                    log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
-                        locNodeId + ", clientNodeId=" + nodeId + ']');
-
-                if (wrk != null)
-                    wrk.addMessage(msg);
-            }
-            else
-                msgWorker.addMessage(msg);
-        }
-
         /**
          * Processes client heartbeat message.
          *
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index 1fa7139e7e956..cb85dc1c44b86 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -292,7 +292,7 @@ protected static String threadStatus(Thread t) {
      *
      * @return Worker thread.
      */
-    protected abstract Collection<IgniteSpiThread> threads();
+    protected abstract IgniteSpiThread workerThread();
 
     /**
      * @throws IgniteSpiException If failed.
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 819108148cf16..419497753bbc2 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -62,8 +62,8 @@
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
 import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
-import org.apache.ignite.spi.IgniteSpiThread;
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryClientReconnectMessage;
@@ -82,14 +82,13 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
-import static org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.DFLT_HEARTBEAT_FREQ;
 
 /**
  * Client-based discovery tests.
  */
 public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     /** */
-    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
     /** */
     protected static final AtomicInteger srvIdx = new AtomicInteger();
@@ -124,9 +123,6 @@ public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     /** */
     private static CountDownLatch clientFailedLatch;
 
-    /** */
-    private static CountDownLatch clientReconnectedLatch;
-
     /** */
     private static CountDownLatch msgLatch;
 
@@ -142,9 +138,6 @@ public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     /** */
     protected long netTimeout = TcpDiscoverySpi.DFLT_NETWORK_TIMEOUT;
 
-    /** */
-    protected Integer reconnectCnt;
-
     /** */
     private boolean longSockTimeouts;
 
@@ -215,9 +208,6 @@ else if (gridName.startsWith("client")) {
         disco.setJoinTimeout(joinTimeout);
         disco.setNetworkTimeout(netTimeout);
 
-        if (reconnectCnt != null)
-            disco.setReconnectCount(reconnectCnt);
-
         disco.setClientReconnectDisabled(reconnectDisabled);
 
         if (disco instanceof TestTcpDiscoverySpi)
@@ -264,8 +254,6 @@ protected TcpDiscoverySpi getDiscoverySpi() {
         clientIpFinder = null;
         joinTimeout = TcpDiscoverySpi.DFLT_JOIN_TIMEOUT;
         netTimeout = TcpDiscoverySpi.DFLT_NETWORK_TIMEOUT;
-        maxMissedClientHbs = TcpDiscoverySpi.DFLT_MAX_MISSED_CLIENT_HEARTBEATS;
-        reconnectCnt = null;
         longSockTimeouts = false;
 
         assert G.allGrids().isEmpty();
@@ -546,176 +534,6 @@ public void testClientReconnectOnRouterFail() throws Exception {
         checkNodes(2, 3);
     }
 
-    /**
-     * Client should reconnect to available server without EVT_CLIENT_NODE_RECONNECTED event.
-     *
-     * @throws Exception If failed.
-     */
-    public void testClientReconnectOnRouterSuspend() throws Exception {
-        reconnectAfterSuspend(false);
-    }
-
-    /**
-     * Client should receive all topology updates after reconnect.
-     *
-     * @throws Exception If failed.
-     */
-    public void testClientReconnectOnRouterSuspendTopologyChange() throws Exception {
-        reconnectAfterSuspend(true);
-    }
-
-    /**
-     * @param changeTop If {@code true} topology is changed after client disconnects
-     * @throws Exception if failed.
-     */
-    private void reconnectAfterSuspend(boolean changeTop) throws Exception {
-        reconnectCnt = 2;
-
-        startServerNodes(2);
-
-        Ignite srv0 = G.ignite("server-0");
-        TcpDiscoveryNode srv0Node = (TcpDiscoveryNode)srv0.cluster().localNode();
-
-        TcpDiscoveryNode srv1Node = (TcpDiscoveryNode)G.ignite("server-1").cluster().localNode();
-
-        clientIpFinder = new TcpDiscoveryVmIpFinder();
-
-        clientIpFinder.setAddresses(
-            Collections.singleton("localhost:" + srv0Node.discoveryPort()));
-
-        startClientNodes(1);
-
-        Ignite client = G.ignite("client-0");
-        TcpDiscoveryNode clientNode = (TcpDiscoveryNode)client.cluster().localNode();
-        TestTcpDiscoverySpi clientSpi = (TestTcpDiscoverySpi)client.configuration().getDiscoverySpi();
-
-        UUID clientNodeId = clientNode.id();
-
-        checkNodes(2, 1);
-
-        clientIpFinder.setAddresses(Collections.singleton("localhost:" + srv1Node.discoveryPort()));
-
-        srvFailedLatch = new CountDownLatch(1);
-
-        attachListeners(2, 1);
-
-        log.info("Pausing router");
-
-        TestTcpDiscoverySpi srvSpi = (TestTcpDiscoverySpi)srv0.configuration().getDiscoverySpi();
-
-        int joinedNodesNum = 3;
-        final CountDownLatch srvJoinedLatch = new CountDownLatch(joinedNodesNum);
-
-        if (changeTop) {
-            client.events().localListen(new IgnitePredicate<Event>() {
-                @Override public boolean apply(Event e) {
-                    srvJoinedLatch.countDown();
-
-                    return true;
-                }
-            }, EVT_NODE_JOINED);
-        }
-
-        srvSpi.pauseAll(true);
-
-        if (changeTop)
-            startServerNodes(joinedNodesNum);
-
-        try {
-            await(srvFailedLatch, 60_000);
-
-            if (changeTop)
-                await(srvJoinedLatch, 5000);
-
-            assertEquals("connected", clientSpi.getSpiState());
-            assertEquals(clientNodeId, clientNode.id());
-            assertEquals(srv1Node.id(), clientNode.clientRouterNodeId());
-        }
-        finally {
-            srvSpi.resumeAll();
-        }
-    }
-
-    /**
-     *
-     */
-    public void testReconnectAfterPause() throws Exception {
-        maxMissedClientHbs = 2;
-
-        startServerNodes(2);
-        startClientNodes(1);
-
-        Ignite client = G.ignite("client-0");
-        TestTcpDiscoverySpi clientSpi = (TestTcpDiscoverySpi)client.configuration().getDiscoverySpi();
-
-        clientReconnectedLatch = new CountDownLatch(1);
-
-        attachListeners(0, 1);
-
-        clientSpi.pauseAll(false);
-
-        try {
-            clientSpi.brakeConnection();
-
-            Thread.sleep(maxMissedClientHbs * DFLT_HEARTBEAT_FREQ * 2);
-        }
-        finally {
-            clientSpi.resumeAll();
-        }
-
-        await(clientReconnectedLatch);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testReconnectAfterMassiveTopologyChange() throws Exception {
-        clientIpFinder = IP_FINDER;
-
-        maxMissedClientHbs = 100;
-        netTimeout = 100000;
-
-        int initSrvsNum = 5;
-        int killNum = 2;
-        int iterations = 3;
-
-        startServerNodes(initSrvsNum);
-        startClientNodes(1);
-
-        Ignite client = G.ignite("client-0");
-        TcpDiscoveryNode clientNode = (TcpDiscoveryNode)client.cluster().localNode();
-        TestTcpDiscoverySpi clientSpi = (TestTcpDiscoverySpi)client.configuration().getDiscoverySpi();
-        final UUID clientNodeId = clientNode.id();
-
-        final CountDownLatch srvJoinedLatch = new CountDownLatch(iterations * killNum);
-
-        client.events().localListen(new IgnitePredicate<Event>() {
-            @Override public boolean apply(Event e) {
-                srvJoinedLatch.countDown();
-
-                return true;
-            }
-        }, EVT_NODE_JOINED);
-
-        int minAliveSrvId = 0;
-
-        for (int i = 0; i < iterations; i++) {
-            startServerNodes(killNum);
-
-            for (int j = 0; j < killNum; j++) {
-                failServer(minAliveSrvId);
-
-                minAliveSrvId++;
-            }
-
-            Thread.sleep(500);
-        }
-
-        await(srvJoinedLatch);
-        assertEquals("connected", clientSpi.getSpiState());
-        assertEquals(clientNodeId, clientNode.id());
-    }
-
     /**
      * @throws Exception If failed.
      */
@@ -1569,16 +1387,17 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
 
         srvSpi.failNode(client.cluster().localNode().id(), null);
 
-        assertTrue(disconnectLatch.await(5000, MILLISECONDS));
-        assertTrue(failLatch.await(5000, MILLISECONDS));
-
         if (changeTop) {
-            startServerNodes(1);
+            Ignite g = startGrid("server-" + srvIdx.getAndIncrement());
+
+            srvNodeIds.add(g.cluster().localNode().id());
 
             clientSpi.resumeAll();
         }
 
+        assertTrue(disconnectLatch.await(5000, MILLISECONDS));
         assertTrue(reconnectLatch.await(5000, MILLISECONDS));
+        assertTrue(failLatch.await(5000, MILLISECONDS));
         assertTrue(joinLatch.await(5000, MILLISECONDS));
 
         long topVer = changeTop ? 5L : 4L;
@@ -2184,20 +2003,6 @@ private void attachListeners(int srvCnt, int clientCnt) throws Exception {
                 }, EVT_NODE_FAILED);
             }
         }
-
-        if (clientReconnectedLatch != null) {
-            for (int i = 0; i < clientCnt; i++) {
-                G.ignite("client-" + i).events().localListen(new IgnitePredicate<Event>() {
-                    @Override public boolean apply(Event evt) {
-                        info("Reconnected event fired on client: " + evt);
-
-                        clientReconnectedLatch.countDown();
-
-                        return true;
-                    }
-                }, EVT_CLIENT_NODE_RECONNECTED);
-            }
-        }
     }
 
     /**
@@ -2267,16 +2072,7 @@ else if (srvNodeIds.contains(id))
      * @throws InterruptedException If interrupted.
      */
     protected void await(CountDownLatch latch) throws InterruptedException {
-        await(latch, awaitTime());
-    }
-
-    /**
-     * @param latch Latch.
-     * @param timeout Timeout.
-     * @throws InterruptedException If interrupted.
-     */
-    protected void await(CountDownLatch latch, long timeout) throws InterruptedException {
-        assertTrue("Latch count: " + latch.getCount(), latch.await(timeout, MILLISECONDS));
+        assertTrue("Latch count: " + latch.getCount(), latch.await(awaitTime(), MILLISECONDS));
     }
 
     /**
@@ -2487,10 +2283,8 @@ public void pauseSocketWrite() {
         public void pauseAll(boolean suspend) {
             pauseResumeOperation(true, openSockLock, writeLock);
 
-            if (suspend) {
-                for (Thread t : impl.threads())
-                    t.suspend();
-            }
+            if (suspend)
+                impl.workerThread().suspend();
         }
 
         /**
@@ -2499,8 +2293,7 @@ public void pauseAll(boolean suspend) {
         public void resumeAll() {
             pauseResumeOperation(false, openSockLock, writeLock);
 
-            for (IgniteSpiThread t : impl.threads())
-                t.resume();
+            impl.workerThread().resume();
         }
 
         /** {@inheritDoc} */

From b541fdeedc1c6cff5c233c71ebc622f2c01c7824 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Fri, 27 Oct 2017 14:12:36 +0300
Subject: [PATCH 410/516] ignite-5860 Try process
 TcpDiscoveryClientReconnectMessage from socket reader instead of always
 processing it on coordinator. (cherry picked from commit 56a63f8)

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  52 ++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 311 +++++++++---------
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   |   4 +-
 .../tcp/TcpClientDiscoverySpiSelfTest.java    | 272 ++++++++++++++-
 4 files changed, 459 insertions(+), 180 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index c32c8e057d9fd..3657b514d2780 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -26,6 +26,7 @@
 import java.net.SocketTimeoutException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -467,7 +468,8 @@ else if (state == DISCONNECTED) {
     }
 
     /**
-     * @param recon {@code True} if reconnects.
+     * @param prevAddr If reconnect is in progress, then previous address of the router the client was connected to
+     *      and {@code null} otherwise.
      * @param timeout Timeout.
      * @return Opened socket or {@code null} if timeout.
      * @throws InterruptedException If interrupted.
@@ -475,9 +477,9 @@ else if (state == DISCONNECTED) {
      * @see TcpDiscoverySpi#joinTimeout
      */
     @SuppressWarnings("BusyWait")
-    @Nullable private T2<SocketStream, Boolean> joinTopology(boolean recon, long timeout)
+    @Nullable private T2<SocketStream, Boolean> joinTopology(InetSocketAddress prevAddr, long timeout)
         throws IgniteSpiException, InterruptedException {
-        Collection<InetSocketAddress> addrs = null;
+        List<InetSocketAddress> addrs = null;
 
         long startTime = U.currentTimeMillis();
 
@@ -486,7 +488,7 @@ else if (state == DISCONNECTED) {
                 throw new InterruptedException();
 
             while (addrs == null || addrs.isEmpty()) {
-                addrs = spi.resolvedAddresses();
+                addrs = new ArrayList<>(spi.resolvedAddresses());
 
                 if (!F.isEmpty(addrs)) {
                     if (log.isDebugEnabled())
@@ -506,17 +508,25 @@ else if (state == DISCONNECTED) {
                 }
             }
 
-            Collection<InetSocketAddress> addrs0 = new ArrayList<>(addrs);
+            // Process failed node last.
+            if (prevAddr != null) {
+                int idx = addrs.indexOf(prevAddr);
 
-            Iterator<InetSocketAddress> it = addrs.iterator();
+                if (idx != -1)
+                    Collections.swap(addrs, idx, 0);
+            }
+
+            Collection<InetSocketAddress> addrs0 = new ArrayList<>(addrs);
 
             boolean wait = false;
 
-            while (it.hasNext()) {
+            for (int i = addrs.size() - 1; i >= 0; i--) {
                 if (Thread.currentThread().isInterrupted())
                     throw new InterruptedException();
 
-                InetSocketAddress addr = it.next();
+                InetSocketAddress addr = addrs.get(i);
+
+                boolean recon = prevAddr != null;
 
                 T3<SocketStream, Integer, Boolean> sockAndRes;
 
@@ -530,7 +540,7 @@ else if (state == DISCONNECTED) {
                 }
 
                 if (sockAndRes == null) {
-                    it.remove();
+                    addrs.remove(i);
 
                     continue;
                 }
@@ -852,8 +862,8 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
     }
 
     /** {@inheritDoc} */
-    @Override protected IgniteSpiThread workerThread() {
-        return msgWorker;
+    @Override protected Collection<IgniteSpiThread> threads() {
+        return Arrays.asList(sockWriter, msgWorker);
     }
 
     /**
@@ -1339,15 +1349,20 @@ private class Reconnector extends IgniteSpiThread {
         private boolean clientAck;
 
         /** */
-        private boolean join;
+        private final boolean join;
+
+        /** */
+        private final InetSocketAddress prevAddr;
 
         /**
          * @param join {@code True} if reconnects during join.
+         * @param prevAddr Address of the node, that this client was previously connected to.
          */
-        protected Reconnector(boolean join) {
+        protected Reconnector(boolean join, InetSocketAddress prevAddr) {
             super(spi.ignite().name(), "tcp-client-disco-reconnector", log);
 
             this.join = join;
+            this.prevAddr = prevAddr;
         }
 
         /**
@@ -1377,7 +1392,7 @@ public void cancel() {
 
             try {
                 while (true) {
-                    T2<SocketStream, Boolean> joinRes = joinTopology(true, timeout);
+                    T2<SocketStream, Boolean> joinRes = joinTopology(prevAddr, timeout);
 
                     if (joinRes == null) {
                         if (join) {
@@ -1612,6 +1627,10 @@ else if (msg instanceof TcpDiscoveryNodeFailedMessage &&
                     }
                     else if (msg instanceof SocketClosedMessage) {
                         if (((SocketClosedMessage)msg).sock == currSock) {
+                            Socket sock = currSock.sock;
+
+                            InetSocketAddress prevAddr = new InetSocketAddress(sock.getInetAddress(), sock.getPort());
+
                             currSock = null;
 
                             boolean join = joinLatch.getCount() > 0;
@@ -1640,8 +1659,7 @@ else if (msg instanceof SocketClosedMessage) {
 
                                     assert reconnector == null;
 
-                                    final Reconnector reconnector = new Reconnector(join);
-                                    this.reconnector = reconnector;
+                                    reconnector = new Reconnector(join, prevAddr);
                                     reconnector.start();
                                 }
                             }
@@ -1812,7 +1830,7 @@ private void tryJoin() throws InterruptedException {
 
             T2<SocketStream, Boolean> joinRes;
             try {
-                joinRes = joinTopology(false, spi.joinTimeout);
+                joinRes = joinTopology(null, spi.joinTimeout);
             }
             catch (IgniteSpiException e) {
                 joinError(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 189b37a6a0899..0f85e0612b942 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -222,6 +222,9 @@ class ServerImpl extends TcpDiscoveryImpl {
     /** Pending custom messages that should not be sent between NodeAdded and NodeAddFinished messages. */
     private Queue<TcpDiscoveryCustomEventMessage> pendingCustomMsgs = new ArrayDeque<>();
 
+    /** Messages history used for client reconnect. */
+    private final EnsuredMessageHistory msgHist = new EnsuredMessageHistory();
+
     /** If non-shared IP finder is used this flag shows whether IP finder contains local address. */
     private boolean ipFinderHasLocAddr;
 
@@ -1642,8 +1645,23 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
     }
 
     /** {@inheritDoc} */
-    @Override protected IgniteSpiThread workerThread() {
-        return msgWorker;
+    @Override protected Collection<IgniteSpiThread> threads() {
+        Collection<IgniteSpiThread> threads;
+
+        synchronized (mux) {
+            threads = new ArrayList<>(readers.size() + clientMsgWorkers.size() + 4);
+            threads.addAll(readers);
+        }
+
+        threads.addAll(clientMsgWorkers.values());
+        threads.add(tcpSrvr);
+        threads.add(ipFinderCleaner);
+        threads.add(msgWorker);
+        threads.add(statsPrinter);
+
+        threads.removeAll(Collections.<IgniteSpiThread>singleton(null));
+
+        return threads;
     }
 
     /**
@@ -2091,7 +2109,9 @@ else if (msg instanceof TcpDiscoveryNodeLeftMessage)
             else if (msg instanceof TcpDiscoveryNodeFailedMessage)
                 clearClientAddFinished(((TcpDiscoveryNodeFailedMessage)msg).failedNodeId());
 
-            msgs.add(msg);
+            synchronized (msgs) {
+                msgs.add(msg);
+            }
         }
 
         /**
@@ -2153,14 +2173,16 @@ private boolean mapsEqual(Map<Integer, byte[]> m1, Map<Integer, byte[]> m2) {
                 // Client connection failed before it received TcpDiscoveryNodeAddedMessage.
                 List<TcpDiscoveryAbstractMessage> res = null;
 
-                for (TcpDiscoveryAbstractMessage msg : msgs) {
-                    if (msg instanceof TcpDiscoveryNodeAddedMessage) {
-                        if (node.id().equals(((TcpDiscoveryNodeAddedMessage)msg).node().id()))
-                            res = new ArrayList<>(msgs.size());
-                    }
+                synchronized (msgs) {
+                    for (TcpDiscoveryAbstractMessage msg : msgs) {
+                        if (msg instanceof TcpDiscoveryNodeAddedMessage) {
+                            if (node.id().equals(((TcpDiscoveryNodeAddedMessage)msg).node().id()))
+                                res = new ArrayList<>(msgs.size());
+                        }
 
-                    if (res != null)
-                        res.add(prepare(msg, node.id()));
+                        if (res != null)
+                            res.add(prepare(msg, node.id()));
+                    }
                 }
 
                 if (log.isDebugEnabled()) {
@@ -2173,20 +2195,26 @@ private boolean mapsEqual(Map<Integer, byte[]> m1, Map<Integer, byte[]> m2) {
                 return res;
             }
             else {
-                if (msgs.isEmpty())
-                    return Collections.emptyList();
+                Collection<TcpDiscoveryAbstractMessage> cp;
 
-                Collection<TcpDiscoveryAbstractMessage> cp = new ArrayList<>(msgs.size());
+                boolean skip;
 
-                boolean skip = true;
+                synchronized (msgs) {
+                    if (msgs.isEmpty())
+                        return Collections.emptyList();
 
-                for (TcpDiscoveryAbstractMessage msg : msgs) {
-                    if (skip) {
-                        if (msg.id().equals(lastMsgId))
-                            skip = false;
+                    cp = new ArrayList<>(msgs.size());
+
+                    skip = true;
+
+                    for (TcpDiscoveryAbstractMessage msg : msgs) {
+                        if (skip) {
+                            if (msg.id().equals(lastMsgId))
+                                skip = false;
+                        }
+                        else
+                            cp.add(prepare(msg, node.id()));
                     }
-                    else
-                        cp.add(prepare(msg, node.id()));
                 }
 
                 cp = !skip ? cp : null;
@@ -2475,9 +2503,6 @@ private class RingMessageWorker extends MessageWorkerAdapter<TcpDiscoveryAbstrac
         /** Pending messages. */
         private final PendingMessages pendingMsgs = new PendingMessages();
 
-        /** Messages history used for client reconnect. */
-        private final EnsuredMessageHistory msgHist = new EnsuredMessageHistory();
-
         /** Last message that updated topology. */
         private TcpDiscoveryAbstractMessage lastMsg;
 
@@ -2649,8 +2674,10 @@ private void initConnectionCheckFrequency() {
             if (msg instanceof TcpDiscoveryJoinRequestMessage)
                 processJoinRequestMessage((TcpDiscoveryJoinRequestMessage)msg);
 
-            else if (msg instanceof TcpDiscoveryClientReconnectMessage)
-                processClientReconnectMessage((TcpDiscoveryClientReconnectMessage)msg);
+            else if (msg instanceof TcpDiscoveryClientReconnectMessage) {
+                if (sendMessageToRemotes(msg))
+                    sendMessageAcrossRing(msg);
+            }
 
             else if (msg instanceof TcpDiscoveryNodeAddedMessage)
                 processNodeAddedMessage((TcpDiscoveryNodeAddedMessage)msg);
@@ -2685,9 +2712,6 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage)
             else
                 assert false : "Unknown message type: " + msg.getClass().getSimpleName();
 
-            if (ensured && redirectToClients(msg))
-                msgHist.add(msg);
-
             if (msg.senderNodeId() != null && !msg.senderNodeId().equals(getLocalNodeId())) {
                 // Received a message from remote node.
                 onMessageExchanged();
@@ -2720,6 +2744,9 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage)
          */
         private void sendMessageToClients(TcpDiscoveryAbstractMessage msg) {
             if (redirectToClients(msg)) {
+                if (spi.ensured(msg))
+                    msgHist.add(msg);
+
                 byte[] msgBytes = null;
 
                 for (ClientMessageWorker clientMsgWorker : clientMsgWorkers.values()) {
@@ -3871,9 +3898,6 @@ else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
                 nodeAddedMsg.client(msg.client());
 
                 processNodeAddedMessage(nodeAddedMsg);
-
-                if (nodeAddedMsg.verified())
-                    msgHist.add(nodeAddedMsg);
             }
             else if (sendMessageToRemotes(msg))
                 sendMessageAcrossRing(msg);
@@ -3961,98 +3985,6 @@ private void trySendMessageDirectly(TcpDiscoveryNode node, TcpDiscoveryAbstractM
                 throw ex;
         }
 
-        /**
-         * Processes client reconnect message.
-         *
-         * @param msg Client reconnect message.
-         */
-        private void processClientReconnectMessage(TcpDiscoveryClientReconnectMessage msg) {
-            UUID nodeId = msg.creatorNodeId();
-
-            UUID locNodeId = getLocalNodeId();
-
-            boolean isLocNodeRouter = locNodeId.equals(msg.routerNodeId());
-
-            if (!msg.verified()) {
-                TcpDiscoveryNode node = ring.node(nodeId);
-
-                assert node == null || node.isClient();
-
-                if (node != null) {
-                    node.clientRouterNodeId(msg.routerNodeId());
-                    node.aliveCheck(spi.maxMissedClientHbs);
-                }
-
-                if (isLocalNodeCoordinator()) {
-                    msg.verify(locNodeId);
-
-                    if (node != null) {
-                        Collection<TcpDiscoveryAbstractMessage> pending = msgHist.messages(msg.lastMessageId(), node);
-
-                        if (pending != null) {
-                            msg.pendingMessages(pending);
-                            msg.success(true);
-
-                            if (log.isDebugEnabled())
-                                log.debug("Accept client reconnect, restored pending messages " +
-                                    "[locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
-                        }
-                        else {
-                            if (log.isDebugEnabled())
-                                log.debug("Failing reconnecting client node because failed to restore pending " +
-                                    "messages [locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
-
-                            TcpDiscoveryNodeFailedMessage nodeFailedMsg = new TcpDiscoveryNodeFailedMessage(locNodeId,
-                                node.id(), node.internalOrder());
-
-                            processNodeFailedMessage(nodeFailedMsg);
-
-                            if (nodeFailedMsg.verified())
-                                msgHist.add(nodeFailedMsg);
-                        }
-                    }
-                    else if (log.isDebugEnabled())
-                        log.debug("Reconnecting client node is already failed [nodeId=" + nodeId + ']');
-
-                    if (isLocNodeRouter) {
-                        ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
-
-                        if (wrk != null)
-                            wrk.addMessage(msg);
-                        else if (log.isDebugEnabled())
-                            log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
-                                locNodeId + ", clientNodeId=" + nodeId + ']');
-                    }
-                    else {
-                        if (sendMessageToRemotes(msg))
-                            sendMessageAcrossRing(msg);
-                    }
-                }
-                else {
-                    if (sendMessageToRemotes(msg))
-                        sendMessageAcrossRing(msg);
-                }
-            }
-            else {
-                if (isLocalNodeCoordinator())
-                    addMessage(new TcpDiscoveryDiscardMessage(locNodeId, msg.id(), false));
-
-                if (isLocNodeRouter) {
-                    ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
-
-                    if (wrk != null)
-                        wrk.addMessage(msg);
-                    else if (log.isDebugEnabled())
-                        log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
-                            locNodeId + ", clientNodeId=" + nodeId + ']');
-                }
-                else {
-                    if (ring.hasRemoteNodes() && !isLocalNodeCoordinator())
-                        sendMessageAcrossRing(msg);
-                }
-            }
-        }
-
         /**
          * Processes node added message.
          *
@@ -4093,9 +4025,6 @@ private void processNodeAddedMessage(TcpDiscoveryNodeAddedMessage msg) {
 
                     processNodeAddFinishedMessage(addFinishMsg);
 
-                    if (addFinishMsg.verified())
-                        msgHist.add(addFinishMsg);
-
                     addMessage(new TcpDiscoveryDiscardMessage(locNodeId, msg.id(), false));
 
                     return;
@@ -5156,9 +5085,6 @@ private void processHeartbeatMessage(TcpDiscoveryHeartbeatMessage msg) {
                                             locNodeId, clientNode.id(), clientNode.internalOrder());
 
                                         processNodeFailedMessage(nodeFailedMsg);
-
-                                        if (nodeFailedMsg.verified())
-                                            msgHist.add(nodeFailedMsg);
                                     }
                                 }
                             }
@@ -5356,9 +5282,6 @@ private void processCustomMessage(TcpDiscoveryCustomEventMessage msg) {
                                 ackMsg.topologyVersion(msg.topologyVersion());
 
                                 processCustomMessage(ackMsg);
-
-                                if (ackMsg.verified())
-                                    msgHist.add(ackMsg);
                             }
                             catch (IgniteCheckedException e) {
                                 U.error(log, "Failed to marshal discovery custom message.", e);
@@ -5460,12 +5383,8 @@ private void checkPendingCustomMessages() {
             if (joiningEmpty && isLocalNodeCoordinator()) {
                 TcpDiscoveryCustomEventMessage msg;
 
-                while ((msg = pollPendingCustomeMessage()) != null) {
+                while ((msg = pollPendingCustomeMessage()) != null)
                     processCustomMessage(msg);
-
-                    if (msg.verified())
-                        msgHist.add(msg);
-                }
             }
         }
 
@@ -6010,24 +5929,22 @@ else if (msg instanceof TcpDiscoveryJoinRequestMessage) {
                             }
                         }
                         else if (msg instanceof TcpDiscoveryClientReconnectMessage) {
-                            if (clientMsgWrk != null) {
-                                TcpDiscoverySpiState state = spiStateCopy();
+                            TcpDiscoverySpiState state = spiStateCopy();
 
-                                if (state == CONNECTED) {
-                                    spi.writeToSocket(msg, sock, RES_OK, sockTimeout);
+                            if (state == CONNECTED) {
+                                spi.writeToSocket(msg, sock, RES_OK, sockTimeout);
 
-                                    if (clientMsgWrk.getState() == State.NEW)
-                                        clientMsgWrk.start();
+                                if (clientMsgWrk != null && clientMsgWrk.getState() == State.NEW)
+                                    clientMsgWrk.start();
 
-                                    msgWorker.addMessage(msg);
+                                processClientReconnectMessage((TcpDiscoveryClientReconnectMessage)msg);
 
-                                    continue;
-                                }
-                                else {
-                                    spi.writeToSocket(msg, sock, RES_CONTINUE_JOIN, sockTimeout);
+                                continue;
+                            }
+                            else {
+                                spi.writeToSocket(msg, sock, RES_CONTINUE_JOIN, sockTimeout);
 
-                                    break;
-                                }
+                                break;
                             }
                         }
                         else if (msg instanceof TcpDiscoveryDuplicateIdMessage) {
@@ -6256,6 +6173,100 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage) {
             }
         }
 
+        /**
+         * Processes client reconnect message.
+         *
+         * @param msg Client reconnect message.
+         */
+        private void processClientReconnectMessage(TcpDiscoveryClientReconnectMessage msg) {
+            UUID nodeId = msg.creatorNodeId();
+
+            UUID locNodeId = getLocalNodeId();
+
+            boolean isLocNodeRouter = msg.routerNodeId().equals(locNodeId);
+
+            TcpDiscoveryNode node = ring.node(nodeId);
+
+            assert node == null || node.isClient();
+
+            if (node != null) {
+                node.clientRouterNodeId(msg.routerNodeId());
+                node.aliveCheck(spi.maxMissedClientHbs);
+            }
+
+            if (!msg.verified()) {
+                if (isLocNodeRouter || isLocalNodeCoordinator()) {
+                    if (node != null) {
+                        Collection<TcpDiscoveryAbstractMessage> pending = msgHist.messages(msg.lastMessageId(), node);
+
+                        if (pending != null) {
+                            msg.verify(locNodeId);
+                            msg.pendingMessages(pending);
+                            msg.success(true);
+
+                            if (log.isDebugEnabled())
+                                log.debug("Accept client reconnect, restored pending messages " +
+                                    "[locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
+                        }
+                        else if (!isLocalNodeCoordinator()) {
+                            if (log.isDebugEnabled())
+                                log.debug("Failed to restore pending messages for reconnecting client. " +
+                                    "Forwarding reconnection message to coordinator " +
+                                    "[locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
+                        }
+                        else {
+                            msg.verify(locNodeId);
+
+                            if (log.isDebugEnabled())
+                                log.debug("Failing reconnecting client node because failed to restore pending " +
+                                    "messages [locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
+
+                            TcpDiscoveryNodeFailedMessage nodeFailedMsg = new TcpDiscoveryNodeFailedMessage(locNodeId,
+                                node.id(), node.internalOrder());
+
+                            msgWorker.addMessage(nodeFailedMsg);
+                        }
+                    }
+                    else {
+                        msg.verify(locNodeId);
+
+                        if (log.isDebugEnabled())
+                            log.debug("Reconnecting client node is already failed [nodeId=" + nodeId + ']');
+                    }
+
+                    if (msg.verified() && isLocNodeRouter) {
+                        ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
+
+                        if (wrk != null)
+                            wrk.addMessage(msg);
+                        else if (log.isDebugEnabled())
+                            log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
+                                locNodeId + ", clientNodeId=" + nodeId + ']');
+                    }
+                    else
+                        msgWorker.addMessage(msg);
+                }
+                else
+                    msgWorker.addMessage(msg);
+            }
+            else {
+                if (isLocalNodeCoordinator())
+                    msgWorker.addMessage(new TcpDiscoveryDiscardMessage(locNodeId, msg.id(), false));
+
+                if (isLocNodeRouter) {
+                    ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
+
+                    if (wrk != null)
+                        wrk.addMessage(msg);
+                    else if (log.isDebugEnabled())
+                        log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
+                            locNodeId + ", clientNodeId=" + nodeId + ']');
+                }
+                else if (ring.hasRemoteNodes() && !isLocalNodeCoordinator())
+                    msgWorker.addMessage(msg);
+            }
+        }
+
         /**
          * Processes client heartbeat message.
          *
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index cb85dc1c44b86..79936f3c3c1f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -290,9 +290,9 @@ protected static String threadStatus(Thread t) {
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      *
-     * @return Worker thread.
+     * @return Worker threads.
      */
-    protected abstract IgniteSpiThread workerThread();
+    protected abstract Collection<IgniteSpiThread> threads();
 
     /**
      * @throws IgniteSpiException If failed.
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 419497753bbc2..190556a2db2d0 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -62,8 +62,8 @@
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.IgniteSpiOperationTimeoutException;
 import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import org.apache.ignite.spi.IgniteSpiThread;
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryClientReconnectMessage;
@@ -88,7 +88,7 @@
  */
 public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     /** */
-    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
     /** */
     protected static final AtomicInteger srvIdx = new AtomicInteger();
@@ -123,6 +123,9 @@ public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     /** */
     private static CountDownLatch clientFailedLatch;
 
+    /** */
+    private static CountDownLatch clientReconnectedLatch;
+
     /** */
     private static CountDownLatch msgLatch;
 
@@ -138,6 +141,9 @@ public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     /** */
     protected long netTimeout = TcpDiscoverySpi.DFLT_NETWORK_TIMEOUT;
 
+    /** */
+    protected Integer reconnectCnt;
+
     /** */
     private boolean longSockTimeouts;
 
@@ -208,6 +214,9 @@ else if (gridName.startsWith("client")) {
         disco.setJoinTimeout(joinTimeout);
         disco.setNetworkTimeout(netTimeout);
 
+        if (reconnectCnt != null)
+            disco.setReconnectCount(reconnectCnt);
+
         disco.setClientReconnectDisabled(reconnectDisabled);
 
         if (disco instanceof TestTcpDiscoverySpi)
@@ -254,6 +263,7 @@ protected TcpDiscoverySpi getDiscoverySpi() {
         clientIpFinder = null;
         joinTimeout = TcpDiscoverySpi.DFLT_JOIN_TIMEOUT;
         netTimeout = TcpDiscoverySpi.DFLT_NETWORK_TIMEOUT;
+        maxMissedClientHbs = TcpDiscoverySpi.DFLT_MAX_MISSED_CLIENT_HEARTBEATS;
         longSockTimeouts = false;
 
         assert G.allGrids().isEmpty();
@@ -534,6 +544,221 @@ public void testClientReconnectOnRouterFail() throws Exception {
         checkNodes(2, 3);
     }
 
+    /**
+     * Client should reconnect to available server without EVT_CLIENT_NODE_RECONNECTED event.
+     *
+     * @throws Exception If failed.
+     */
+    public void testClientReconnectOnRouterSuspend() throws Exception {
+        reconnectAfterSuspend(false);
+    }
+
+    /**
+     * Client should receive all topology updates after reconnect.
+     *
+     * @throws Exception If failed.
+     */
+    public void testClientReconnectOnRouterSuspendTopologyChange() throws Exception {
+        reconnectAfterSuspend(true);
+    }
+
+    /**
+     * @param changeTop If {@code true} topology is changed after client disconnects
+     * @throws Exception if failed.
+     */
+    private void reconnectAfterSuspend(boolean changeTop) throws Exception {
+        reconnectCnt = 2;
+
+        startServerNodes(2);
+
+        Ignite srv0 = grid("server-0");
+        TcpDiscoveryNode srv0Node = (TcpDiscoveryNode)srv0.cluster().localNode();
+
+        TcpDiscoveryNode srv1Node = (TcpDiscoveryNode)grid("server-1").cluster().localNode();
+
+        clientIpFinder = new TcpDiscoveryVmIpFinder();
+
+        clientIpFinder.setAddresses(
+            Collections.singleton("localhost:" + srv0Node.discoveryPort()));
+
+        startClientNodes(1);
+
+        Ignite client = grid("client-0");
+        TcpDiscoveryNode clientNode = (TcpDiscoveryNode)client.cluster().localNode();
+        TestTcpDiscoverySpi clientSpi = (TestTcpDiscoverySpi)client.configuration().getDiscoverySpi();
+
+        UUID clientNodeId = clientNode.id();
+
+        checkNodes(2, 1);
+
+        clientIpFinder.setAddresses(Collections.singleton("localhost:" + srv1Node.discoveryPort()));
+
+        srvFailedLatch = new CountDownLatch(1);
+
+        attachListeners(2, 1);
+
+        log.info("Pausing router");
+
+        TestTcpDiscoverySpi srvSpi = (TestTcpDiscoverySpi)srv0.configuration().getDiscoverySpi();
+
+        int joinedNodesNum = 3;
+        final CountDownLatch srvJoinedLatch = new CountDownLatch(joinedNodesNum);
+
+        if (changeTop) {
+            client.events().localListen(new IgnitePredicate<Event>() {
+                @Override public boolean apply(Event e) {
+                    srvJoinedLatch.countDown();
+
+                    return true;
+                }
+            }, EVT_NODE_JOINED);
+        }
+
+        srvSpi.pauseAll(true);
+
+        if (changeTop)
+            startServerNodes(joinedNodesNum);
+
+        try {
+            await(srvFailedLatch, 60_000);
+
+            if (changeTop)
+                await(srvJoinedLatch, 5000);
+
+            assertEquals("connected", clientSpi.getSpiState());
+            assertEquals(clientNodeId, clientNode.id());
+            assertEquals(srv1Node.id(), clientNode.clientRouterNodeId());
+        }
+        finally {
+            srvSpi.resumeAll();
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testClientReconnectHistoryMissingOnRouter() throws Exception {
+        maxMissedClientHbs = 30;
+        netTimeout = 60000;
+
+        startServerNodes(2);
+
+        Ignite srv0 = grid("server-0");
+        TcpDiscoveryNode srv0Node = (TcpDiscoveryNode)srv0.cluster().localNode();
+
+        clientIpFinder = new TcpDiscoveryVmIpFinder();
+        clientIpFinder.setAddresses(
+            Collections.singleton("localhost:" + srv0Node.discoveryPort()));
+
+        startClientNodes(1);
+
+        attachListeners(0, 1);
+
+        Ignite client = grid("client-0");
+        TcpDiscoveryNode clientNode = (TcpDiscoveryNode)client.cluster().localNode();
+        TestTcpDiscoverySpi clientSpi = (TestTcpDiscoverySpi)client.configuration().getDiscoverySpi();
+        UUID clientNodeId = clientNode.id();
+
+        checkNodes(2, 1);
+
+        clientSpi.pauseAll(true);
+
+        stopGrid(srv0.name());
+
+        startServerNodes(1);
+
+        Ignite srv2 = grid("server-2");
+        TcpDiscoveryNode srv2Node = (TcpDiscoveryNode)srv2.cluster().localNode();
+        clientIpFinder.setAddresses(
+            Collections.singleton("localhost:" + srv2Node.discoveryPort()));
+
+        clientSpi.resumeAll();
+
+        awaitPartitionMapExchange();
+
+        assertEquals("connected", clientSpi.getSpiState());
+        assertEquals(clientNodeId, clientNode.id());
+        assertEquals(srv2Node.id(), clientNode.clientRouterNodeId());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectAfterPause() throws Exception {
+        startServerNodes(2);
+        startClientNodes(1);
+
+        Ignite client = grid("client-0");
+        TestTcpDiscoverySpi clientSpi = (TestTcpDiscoverySpi)client.configuration().getDiscoverySpi();
+
+        clientReconnectedLatch = new CountDownLatch(1);
+
+        attachListeners(0, 1);
+
+        clientSpi.pauseAll(false);
+
+        try {
+            clientSpi.brakeConnection();
+
+            Thread.sleep(maxMissedClientHbs * clientSpi.getHeartbeatFrequency() * 2);
+        }
+        finally {
+            clientSpi.resumeAll();
+        }
+
+        await(clientReconnectedLatch);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testReconnectAfterMassiveTopologyChange() throws Exception {
+        clientIpFinder = IP_FINDER;
+
+        maxMissedClientHbs = 30;
+        netTimeout = 60000;
+
+        int initSrvsNum = 5;
+        int killNum = 3;
+        int iterations = 10;
+
+        startServerNodes(initSrvsNum);
+        startClientNodes(1);
+
+        Ignite client = grid("client-0");
+        TcpDiscoveryNode clientNode = (TcpDiscoveryNode)client.cluster().localNode();
+        TestTcpDiscoverySpi clientSpi = (TestTcpDiscoverySpi)client.configuration().getDiscoverySpi();
+        final UUID clientNodeId = clientNode.id();
+
+        final CountDownLatch srvJoinedLatch = new CountDownLatch(iterations * killNum);
+
+        client.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event e) {
+                srvJoinedLatch.countDown();
+
+                return true;
+            }
+        }, EVT_NODE_JOINED);
+
+        int minAliveSrvId = 0;
+
+        for (int i = 0; i < iterations; i++) {
+            for (int j = 0; j < killNum; j++) {
+                stopGrid(minAliveSrvId);
+
+                minAliveSrvId++;
+            }
+
+            startServerNodes(killNum);
+
+            awaitPartitionMapExchange();
+        }
+
+        await(srvJoinedLatch);
+        assertEquals("connected", clientSpi.getSpiState());
+        assertEquals(clientNodeId, clientNode.id());
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -1387,17 +1612,16 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
 
         srvSpi.failNode(client.cluster().localNode().id(), null);
 
-        if (changeTop) {
-            Ignite g = startGrid("server-" + srvIdx.getAndIncrement());
+        assertTrue(disconnectLatch.await(5000, MILLISECONDS));
+        assertTrue(failLatch.await(5000, MILLISECONDS));
 
-            srvNodeIds.add(g.cluster().localNode().id());
+        if (changeTop) {
+            startServerNodes(1);
 
             clientSpi.resumeAll();
         }
 
-        assertTrue(disconnectLatch.await(5000, MILLISECONDS));
         assertTrue(reconnectLatch.await(5000, MILLISECONDS));
-        assertTrue(failLatch.await(5000, MILLISECONDS));
         assertTrue(joinLatch.await(5000, MILLISECONDS));
 
         long topVer = changeTop ? 5L : 4L;
@@ -2003,6 +2227,20 @@ private void attachListeners(int srvCnt, int clientCnt) throws Exception {
                 }, EVT_NODE_FAILED);
             }
         }
+
+        if (clientReconnectedLatch != null) {
+            for (int i = 0; i < clientCnt; i++) {
+                G.ignite("client-" + i).events().localListen(new IgnitePredicate<Event>() {
+                    @Override public boolean apply(Event evt) {
+                        info("Reconnected event fired on client: " + evt);
+
+                        clientReconnectedLatch.countDown();
+
+                        return true;
+                    }
+                }, EVT_CLIENT_NODE_RECONNECTED);
+            }
+        }
     }
 
     /**
@@ -2072,7 +2310,16 @@ else if (srvNodeIds.contains(id))
      * @throws InterruptedException If interrupted.
      */
     protected void await(CountDownLatch latch) throws InterruptedException {
-        assertTrue("Latch count: " + latch.getCount(), latch.await(awaitTime(), MILLISECONDS));
+        await(latch, awaitTime());
+    }
+
+    /**
+     * @param latch Latch.
+     * @param timeout Timeout.
+     * @throws InterruptedException If interrupted.
+     */
+    protected void await(CountDownLatch latch, long timeout) throws InterruptedException {
+        assertTrue("Latch count: " + latch.getCount(), latch.await(timeout, MILLISECONDS));
     }
 
     /**
@@ -2283,8 +2530,10 @@ public void pauseSocketWrite() {
         public void pauseAll(boolean suspend) {
             pauseResumeOperation(true, openSockLock, writeLock);
 
-            if (suspend)
-                impl.workerThread().suspend();
+            if (suspend) {
+                for (Thread t : impl.threads())
+                    t.suspend();
+            }
         }
 
         /**
@@ -2293,7 +2542,8 @@ public void pauseAll(boolean suspend) {
         public void resumeAll() {
             pauseResumeOperation(false, openSockLock, writeLock);
 
-            impl.workerThread().resume();
+            for (IgniteSpiThread t : impl.threads())
+                t.resume();
         }
 
         /** {@inheritDoc} */

From 0ecc05526c03f7e1099dfc660971b20916dc66d5 Mon Sep 17 00:00:00 2001
From: Oleg Ostanin <oostanin@gridgain.com>
Date: Tue, 24 Oct 2017 14:53:16 +0300
Subject: [PATCH 411/516] IGNITE-6660 Python Redis example fails for python 3
 run.

---
 examples/config/redis/example-redis.xml | 74 +++++++++++++++++++++++++
 examples/redis/redis-example.py         | 16 +++---
 2 files changed, 82 insertions(+), 8 deletions(-)
 create mode 100644 examples/config/redis/example-redis.xml

diff --git a/examples/config/redis/example-redis.xml b/examples/config/redis/example-redis.xml
new file mode 100644
index 0000000000000..5db27a1848d6f
--- /dev/null
+++ b/examples/config/redis/example-redis.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    Ignite Spring configuration file to startup Ignite cache.
+
+    This file demonstrates how to configure cache using Spring. Provided cache
+    will be created on node startup.
+
+    Use this configuration file when running Redis examples (see 'examples/redis' folder).
+
+    When starting a standalone node, you need to execute the following command:
+    {IGNITE_HOME}/bin/ignite.{bat|sh} examples/config/redis/example-redis.xml
+
+    When starting Ignite from Java IDE, pass path to this file to Ignition:
+    Ignition.start("examples/config/redis/example-redis.xml");
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans.xsd">
+    <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+        <property name="cacheConfiguration">
+            <list>
+                <!-- Partitioned cache example configuration (Atomic mode). -->
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="name" value="redis-ignite-internal-cache-0"/>
+                    <property name="atomicityMode" value="ATOMIC"/>
+                    <property name="backups" value="1"/>
+                </bean>
+            </list>
+        </property>
+
+        <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. -->
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <!--
+                        Ignite provides several options for automatic discovery that can be used
+                        instead os static IP based discovery. For information on all options refer
+                        to our documentation: http://apacheignite.readme.io/docs/cluster-config
+                    -->
+                    <!-- Uncomment static IP finder to enable static-based discovery of initial nodes. -->
+                    <!--<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">-->
+                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
+                        <property name="addresses">
+                            <list>
+                                <!-- In distributed environment, replace with actual host IP address. -->
+                                <value>127.0.0.1:47500..47509</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+    </bean>
+</beans>
diff --git a/examples/redis/redis-example.py b/examples/redis/redis-example.py
index f6c4f367048de..8c72a69a7f309 100644
--- a/examples/redis/redis-example.py
+++ b/examples/redis/redis-example.py
@@ -29,34 +29,34 @@
 r.set('k1', 1)
 
 # check.
-print 'Value for "k1": %s' % r.get('k1')
+print('Value for "k1": %s' % r.get('k1'))
 
 # change entry's value.
 r.set('k1', 'new_val')
 
 # check.
-print 'Value for "k1": %s' % r.get('k1')
+print('Value for "k1": %s' % r.get('k1'))
 
 # set another entry.
 r.set('k2', 2)
 
 # check.
-print 'Value for "k2": %s' % r.get('k2')
+print('Value for "k2": %s' % r.get('k2'))
 
 # get both values.
-print 'Values for "k1" and "k2": %s' % r.mget('k1', 'k2')
+print('Values for "k1" and "k2": %s' % r.mget('k1', 'k2'))
 
 # delete one entry.
 r.delete('k1')
 
 # check one entry left.
-print 'Values for "k1" and "k2": %s' % r.mget('k1', 'k2')
+print('Values for "k1" and "k2": %s' % r.mget('k1', 'k2'))
 
 # check db size
-print 'Db size: %d' % r.dbsize()
+print('Db size: %d' % r.dbsize())
 
 # increment.
-print 'Value for incremented "inc_k" : %s' % r.incr('inc_k')
+print('Value for incremented "inc_k" : %s' % r.incr('inc_k'))
 
 # increment again.
-print 'Value for incremented "inc_k" : %s' % r.incr('inc_k')
+print('Value for incremented "inc_k" : %s' % r.incr('inc_k'))

From 18ed82c6b5cf2674bc98b83f94f7899776dd2334 Mon Sep 17 00:00:00 2001
From: voipp <aleksaero@gmail.com>
Date: Tue, 14 Feb 2017 15:08:59 +0300
Subject: [PATCH 412/516] IGNITE-4492 Add MBean for StripedExecutor. This
 closes #1491.

(cherry picked from commit 8e12513)
---
 .../apache/ignite/internal/IgniteKernal.java  |  48 ++++-
 .../StripedExecutorMXBeanAdapter.java         |  90 ++++++++++
 .../ignite/internal/util/StripedExecutor.java |  55 +++++-
 .../ignite/mxbean/StripedExecutorMXBean.java  |  90 ++++++++++
 .../internal/util/StripedExecutorTest.java    | 168 ++++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |   2 +
 6 files changed, 447 insertions(+), 6 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/StripedExecutorMXBeanAdapter.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/mxbean/StripedExecutorMXBean.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 623072e3e6b53..81c9e548ced4b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -168,6 +168,7 @@
 import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.mxbean.ClusterLocalNodeMetricsMXBean;
 import org.apache.ignite.mxbean.IgniteMXBean;
+import org.apache.ignite.mxbean.StripedExecutorMXBean;
 import org.apache.ignite.mxbean.ThreadPoolMXBean;
 import org.apache.ignite.plugin.IgnitePlugin;
 import org.apache.ignite.plugin.PluginNotFoundException;
@@ -308,6 +309,10 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     @GridToStringExclude
     private ObjectName restExecSvcMBean;
 
+    /** */
+    @GridToStringExclude
+    private ObjectName stripedExecSvcMBean;
+
     /** Kernal start timestamp. */
     private long startTime = U.currentTimeMillis();
 
@@ -1011,6 +1016,7 @@ public void start(
                 restExecSvc,
                 utilityCachePool,
                 marshCachePool);
+            registerStripedExecutorMBean(stripedExecSvc);
 
             // Lifecycle bean notifications.
             notifyLifecycleBeans(AFTER_NODE_START);
@@ -1598,7 +1604,14 @@ private void registerLocalNodeMBean() throws IgniteCheckedException {
         }
     }
 
-    /** @throws IgniteCheckedException If registration failed. */
+    /**
+     * @param execSvc
+     * @param sysExecSvc
+     * @param p2pExecSvc
+     * @param mgmtExecSvc
+     * @param restExecSvc
+     * @throws IgniteCheckedException If failed.
+     */
     private void registerExecutorMBeans(
         ExecutorService execSvc,
         ExecutorService sysExecSvc,
@@ -1648,8 +1661,34 @@ private ObjectName registerExecutorMBean(ExecutorService exec, String name) thro
             return res;
         }
         catch (JMException e) {
-            throw new IgniteCheckedException("Failed to register executor service MBean [name=" + name + ", exec=" + exec + ']',
-                e);
+            throw new IgniteCheckedException("Failed to register executor service MBean [name=" + name +
+                ", exec=" + exec + ']', e);
+        }
+    }
+
+    /**
+     * @param stripedExecSvc Executor service.
+     * @throws IgniteCheckedException If registration failed.
+     */
+    private void registerStripedExecutorMBean(StripedExecutor stripedExecSvc) throws IgniteCheckedException {
+        if (stripedExecSvc != null) {
+            String name = "StripedExecutor";
+
+            try {
+                stripedExecSvcMBean = U.registerMBean(
+                    cfg.getMBeanServer(),
+                    cfg.getGridName(),
+                    "Thread Pools",
+                    name,
+                    new StripedExecutorMXBeanAdapter(stripedExecSvc),
+                    StripedExecutorMXBean.class);
+
+                if (log.isDebugEnabled())
+                    log.debug("Registered executor service MBean: " + stripedExecSvcMBean);
+            } catch (JMException e) {
+                throw new IgniteCheckedException("Failed to register executor service MBean [name="
+                    + name + ", exec=" + stripedExecSvc + ']', e);
+            }
         }
     }
 
@@ -2116,7 +2155,8 @@ else if (state == STARTING)
                     unregisterMBean(locNodeMBean) &
                     unregisterMBean(utilityExecSvcMBean) &
                     unregisterMBean(marshallerExecSvcMBean) &
-                    unregisterMBean(restExecSvcMBean)
+                    unregisterMBean(restExecSvcMBean) &
+                    unregisterMBean(stripedExecSvcMBean)
             ))
                 errOnStop = false;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/StripedExecutorMXBeanAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/StripedExecutorMXBeanAdapter.java
new file mode 100644
index 0000000000000..e6811b7aee762
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/StripedExecutorMXBeanAdapter.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.concurrent.ExecutorService;
+import org.apache.ignite.internal.util.StripedExecutor;
+import org.apache.ignite.mxbean.StripedExecutorMXBean;
+
+/**
+ * Adapter for {@link StripedExecutorMXBean} which delegates all method calls to the underlying
+ * {@link ExecutorService} instance.
+ */
+public class StripedExecutorMXBeanAdapter implements StripedExecutorMXBean {
+    /** */
+    private final StripedExecutor exec;
+
+    /**
+     * @param exec Executor service
+     */
+    StripedExecutorMXBeanAdapter(StripedExecutor exec) {
+        assert exec != null;
+
+        this.exec = exec;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void checkStarvation() {
+        exec.checkStarvation();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getStripesCount() {
+        return exec.stripes();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isShutdown() {
+        return exec.isShutdown();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isTerminated() {
+        return exec.isTerminated();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getTotalQueueSize() {
+        return exec.queueSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTotalCompletedTasksCount() {
+        return exec.completedTasks();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long[] getStripesCompletedTasksCounts() {
+        return exec.stripesCompletedTasks();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getActiveCount() {
+        return exec.activeStripesCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean[] getStripesActiveStatuses() {
+        return exec.stripesActiveStatuses();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int[] getStripesQueueSizes() {
+        return exec.stripesQueueSizes();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
index 201cb34c1e4e8..e70f0ce2b761a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
@@ -55,9 +55,10 @@ public class StripedExecutor implements ExecutorService {
     private final IgniteLogger log;
 
     /**
-     * Constructor.
-     *
      * @param cnt Count.
+     * @param gridName Node name.
+     * @param poolName Pool name.
+     * @param log Logger.
      */
     public StripedExecutor(int cnt, String gridName, String poolName, final IgniteLogger log) {
         A.ensure(cnt > 0, "cnt > 0");
@@ -267,6 +268,56 @@ public long completedTasks() {
         return cnt;
     }
 
+    /**
+     * @return Completed tasks per stripe count.
+     */
+    public long[] stripesCompletedTasks() {
+        long[] res = new long[stripes()];
+
+        for (int i = 0; i < res.length; i++)
+            res[i] = stripes[i].completedCnt;
+
+        return res;
+    }
+
+    /**
+     * @return Number of active tasks per stripe.
+     */
+    public boolean[] stripesActiveStatuses() {
+        boolean[] res = new boolean[stripes()];
+
+        for (int i = 0; i < res.length; i++)
+            res[i] = stripes[i].active;
+
+        return res;
+    }
+
+    /**
+     * @return Number of active tasks.
+     */
+    public int activeStripesCount() {
+        int res = 0;
+
+        for (boolean status : stripesActiveStatuses()) {
+            if (status)
+                res++;
+        }
+
+        return res;
+    }
+
+    /**
+     * @return Size of queue per stripe.
+     */
+    public int[] stripesQueueSizes() {
+        int[] res = new int[stripes()];
+
+        for (int i = 0; i < res.length; i++)
+            res[i] = stripes[i].queueSize();
+
+        return res;
+    }
+
     /**
      * Operation not supported.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/StripedExecutorMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/StripedExecutorMXBean.java
new file mode 100644
index 0000000000000..7428b19600d20
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/StripedExecutorMXBean.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mxbean;
+
+/**
+ * MBean that provides access to information about striped executor service.
+ */
+@MXBeanDescription("MBean that provides access to information about striped executor service.")
+public interface StripedExecutorMXBean {
+    /**
+     * Checks for starvation in striped pool, dumps in log information if potential starvation
+     * was found.
+     */
+    @MXBeanDescription("Checks for starvation in striped pool.")
+    public void checkStarvation();
+
+    /**
+     * @return Stripes count.
+     */
+    @MXBeanDescription("Stripes count.")
+    public int getStripesCount();
+
+    /**
+     *
+     * @return {@code True} if this executor has been shut down.
+     */
+    @MXBeanDescription("True if this executor has been shut down.")
+    public boolean isShutdown();
+
+    /**
+     * Note that
+     * {@code isTerminated()} is never {@code true} unless either {@code shutdown()} or
+     * {@code shutdownNow()} was called first.
+     *
+     * @return {@code True} if all tasks have completed following shut down.
+     */
+    @MXBeanDescription("True if all tasks have completed following shut down.")
+    public boolean isTerminated();
+
+    /**
+     * @return Return total queue size of all stripes.
+     */
+    @MXBeanDescription("Total queue size of all stripes.")
+    public int getTotalQueueSize();
+
+    /**
+     * @return Completed tasks count.
+     */
+    @MXBeanDescription("Completed tasks count of all stripes.")
+    public long getTotalCompletedTasksCount();
+
+    /**
+     * @return Number of completed tasks per stripe.
+     */
+    @MXBeanDescription("Number of completed tasks per stripe.")
+    public long[] getStripesCompletedTasksCounts();
+
+    /**
+     * @return Number of active tasks.
+     */
+    @MXBeanDescription("Number of active tasks of all stripes.")
+    public int getActiveCount();
+
+    /**
+     * @return Number of active tasks per stripe.
+     */
+    @MXBeanDescription("Number of active tasks per stripe.")
+    public boolean[] getStripesActiveStatuses();
+
+    /**
+     * @return Size of queue per stripe.
+     */
+    @MXBeanDescription("Size of queue per stripe.")
+    public int[] getStripesQueueSizes();
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
new file mode 100644
index 0000000000000..543907fcde60a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import org.apache.ignite.logger.java.JavaLogger;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class StripedExecutorTest extends GridCommonAbstractTest {
+    /** */
+    private StripedExecutor stripedExecSvc;
+
+    /** {@inheritDoc} */
+    @Override public void beforeTest() {
+        stripedExecSvc = new StripedExecutor(3, "foo name", "pool name", new JavaLogger());
+    }
+
+    /** {@inheritDoc} */
+    @Override public void afterTest() {
+        stripedExecSvc.shutdown();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCompletedTasks() throws Exception {
+        stripedExecSvc.execute(0, new TestRunnable());
+        stripedExecSvc.execute(1, new TestRunnable());
+
+        sleepASec();
+
+        assertEquals(2, stripedExecSvc.completedTasks());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStripesCompletedTasks() throws Exception {
+        stripedExecSvc.execute(0, new TestRunnable());
+        stripedExecSvc.execute(1, new TestRunnable());
+
+        sleepASec();
+
+        long[] completedTaks = stripedExecSvc.stripesCompletedTasks();
+
+        assertEquals(1, completedTaks[0]);
+        assertEquals(1, completedTaks[1]);
+        assertEquals(0, completedTaks[2]);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStripesActiveStatuses() throws Exception {
+        stripedExecSvc.execute(0, new TestRunnable());
+        stripedExecSvc.execute(1, new TestRunnable(true));
+
+        sleepASec();
+
+        boolean[] statuses = stripedExecSvc.stripesActiveStatuses();
+
+        assertFalse(statuses[0]);
+        assertTrue(statuses[1]);
+        assertFalse(statuses[0]);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testActiveStripesCount() throws Exception {
+        stripedExecSvc.execute(0, new TestRunnable());
+        stripedExecSvc.execute(1, new TestRunnable(true));
+
+        sleepASec();
+
+        assertEquals(1, stripedExecSvc.activeStripesCount());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStripesQueueSizes() throws Exception {
+        stripedExecSvc.execute(0, new TestRunnable());
+        stripedExecSvc.execute(0, new TestRunnable(true));
+        stripedExecSvc.execute(0, new TestRunnable(true));
+        stripedExecSvc.execute(1, new TestRunnable(true));
+        stripedExecSvc.execute(1, new TestRunnable(true));
+        stripedExecSvc.execute(1, new TestRunnable(true));
+
+        sleepASec();
+
+        int[] queueSizes = stripedExecSvc.stripesQueueSizes();
+
+        assertEquals(1, queueSizes[0]);
+        assertEquals(2, queueSizes[1]);
+        assertEquals(0, queueSizes[2]);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testQueueSize() throws Exception {
+        stripedExecSvc.execute(1, new TestRunnable());
+        stripedExecSvc.execute(1, new TestRunnable(true));
+        stripedExecSvc.execute(1, new TestRunnable(true));
+
+        sleepASec();
+
+        assertEquals(1, stripedExecSvc.queueSize());
+    }
+
+    /**
+     *
+     */
+    private final class TestRunnable implements Runnable {
+        /** */
+        private final boolean infinitely;
+
+        /**
+         *
+         */
+        public TestRunnable() {
+            this(false);
+        }
+
+        /**
+         * @param infinitely {@code True} if should sleep infinitely.
+         */
+        public TestRunnable(boolean infinitely) {
+            this.infinitely = infinitely;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            try {
+                while (infinitely)
+                    sleepASec();
+            }
+            catch (InterruptedException e) {
+                info("Got interrupted exception while sleeping: " + e);
+            }
+        }
+    }
+
+    /**
+     * @throws InterruptedException If interrupted.
+     */
+    private void sleepASec() throws InterruptedException {
+        Thread.sleep(1000);
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 9e6c72e0e6f9c..7e299ca9a8cff 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -74,6 +74,7 @@
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointTaskSelfTest;
 import org.apache.ignite.internal.managers.communication.GridCommunicationManagerListenersSelfTest;
 import org.apache.ignite.internal.processors.compute.PublicThreadpoolStarvationTest;
+import org.apache.ignite.internal.util.StripedExecutorTest;
 import org.apache.ignite.p2p.GridMultinodeRedeployContinuousModeSelfTest;
 import org.apache.ignite.p2p.GridMultinodeRedeployIsolatedModeSelfTest;
 import org.apache.ignite.p2p.GridMultinodeRedeployPrivateModeSelfTest;
@@ -156,6 +157,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteRoundRobinErrorAfterClientReconnectTest.class);
         suite.addTestSuite(PublicThreadpoolStarvationTest.class);
         suite.addTestSuite(IgniteComputeJobOneThreadTest.class);
+        suite.addTestSuite(StripedExecutorTest.class);
 
         return suite;
     }

From 0a63367b9926fd216e49ef8614cd3dee6e5d7633 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 27 Oct 2017 15:03:07 +0300
Subject: [PATCH 413/516] Backported IGNITE-5100 Do not remove active exchange
 futures from ExchangeFutureSet.

(cherry picked from commit ecb9e4d)
---
 .../apache/ignite/IgniteSystemProperties.java |  3 +
 .../GridCachePartitionExchangeManager.java    | 15 +++-
 ...ePartitionExchangeManagerHistSizeTest.java | 76 +++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |  3 +
 .../query/h2/DmlStatementsProcessor.java      | 11 +++
 5 files changed, 104 insertions(+), 4 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 3c5a5a6c5d44b..4bae00e4b739c 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -157,6 +157,9 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_CONSOLE_APPENDER = "IGNITE_CONSOLE_APPENDER";
 
+    /** Maximum size for exchange history. Default value is {@code 1000}.*/
+    public static final String IGNITE_EXCHANGE_HISTORY_SIZE = "IGNITE_EXCHANGE_HISTORY_SIZE";
+
     /**
      * Name of the system property defining name of command line program.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 183debd4148d1..93fe6a835053b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -117,7 +117,8 @@
  */
 public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedManagerAdapter<K, V> {
     /** Exchange history size. */
-    private static final int EXCHANGE_HISTORY_SIZE = 1000;
+    private static final int EXCHANGE_HISTORY_SIZE =
+        IgniteSystemProperties.getInteger(IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE, 1_000);
 
     /** Atomic reference for pending timeout object. */
     private AtomicReference<ResendTimeoutObject> pendingResend = new AtomicReference<>();
@@ -1995,8 +1996,14 @@ private ExchangeFutureSet() {
             GridDhtPartitionsExchangeFuture fut) {
             GridDhtPartitionsExchangeFuture cur = super.addx(fut);
 
-            while (size() > EXCHANGE_HISTORY_SIZE)
+            while (size() > EXCHANGE_HISTORY_SIZE) {
+                GridDhtPartitionsExchangeFuture last = last();
+
+                if (!last.isDone())
+                    break;
+
                 removeLast();
+            }
 
             // Return the value in the set.
             return cur == null ? fut : cur;
@@ -2004,8 +2011,8 @@ private ExchangeFutureSet() {
 
         /** {@inheritDoc} */
         @Nullable @Override public synchronized GridDhtPartitionsExchangeFuture removex(
-            GridDhtPartitionsExchangeFuture val
-        ) {
+            GridDhtPartitionsExchangeFuture val) {
+
             return super.removex(val);
         }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java
new file mode 100644
index 0000000000000..934f40d94376f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerHistSizeTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE;
+
+/**
+ * Test exchange history size parameter effect.
+ */
+public class GridCachePartitionExchangeManagerHistSizeTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private String oldHistVal;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setCacheConfiguration(new CacheConfiguration());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        oldHistVal = System.getProperty(IGNITE_EXCHANGE_HISTORY_SIZE);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        if (oldHistVal != null)
+            System.setProperty(IGNITE_EXCHANGE_HISTORY_SIZE, oldHistVal);
+        else
+            System.clearProperty(IGNITE_EXCHANGE_HISTORY_SIZE);
+    }
+
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSingleExchangeHistSize() throws Exception {
+        System.setProperty(IGNITE_EXCHANGE_HISTORY_SIZE, "1");
+
+        startGridsMultiThreaded(10);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 160f85d0a1387..d4a5a8be08e22 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.GridCachePartitionExchangeManagerHistSizeTest;
 import org.apache.ignite.internal.processors.cache.CacheKeepBinaryTransactionTest;
 import org.apache.ignite.internal.processors.cache.CacheNearReaderUpdateTest;
 import org.apache.ignite.internal.processors.cache.CacheRebalancingSelfTest;
@@ -75,6 +76,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(ExternalizableKeyValueTest.class);
 
+        suite.addTestSuite(GridCachePartitionExchangeManagerHistSizeTest.class);
+
         return suite;
     }
 }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
index 60bc483576db5..77c1df526150f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
@@ -116,6 +116,17 @@ public class DmlStatementsProcessor {
         this.indexing = indexing;
     }
 
+    /**
+     * Handle cache stop.
+     *
+     * @param spaceName Cache name.
+     */
+    public void onCacheStop(String spaceName) {
+        spaceName = F.isEmpty(spaceName) ? "default" : spaceName;
+
+        planCache.remove(spaceName);
+    }
+
     /**
      * Execute DML statement, possibly with few re-attempts in case of concurrent data modifications.
      *

From 84ebd019e514c3015fd82d5658e81fcfdaf9a5d4 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 23 Oct 2017 19:39:26 +0300
Subject: [PATCH 414/516] Backported IGNITE-5116: Fixed stale DML plan caching
 in DmlStatementsProcessor.

(cherry picked from commit b7ba1d4)
---
 .../ignite/internal/processors/query/h2/IgniteH2Indexing.java   | 1 +
 .../cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java        | 2 +-
 .../processors/cache/IgniteCacheInsertSqlQuerySelfTest.java     | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 4080c1ff06bfc..14d185091fb4d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -2251,6 +2251,7 @@ private void createSqlFunctions(String schema, Class<?>[] clss) throws IgniteChe
         if (rmv != null) {
             space2schema.remove(emptyIfNull(rmv.spaceName));
             mapQryExec.onCacheStop(ccfg.getName());
+            dmlProc.onCacheStop(rmv.spaceName);
 
             rmv.onDrop();
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
index 4470a48e4df09..2bc8c33306cbd 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
@@ -361,7 +361,7 @@ Object createPerson2(int id, String name, int valFld) {
      * @param idxTypes Indexed types.
      * @return Cache configuration.
      */
-    private static CacheConfiguration cacheConfig(String name, boolean partitioned, boolean escapeSql, Class<?>... idxTypes) {
+    static CacheConfiguration cacheConfig(String name, boolean partitioned, boolean escapeSql, Class<?>... idxTypes) {
         return new CacheConfiguration()
             .setName(name)
             .setCacheMode(partitioned ? CacheMode.PARTITIONED : CacheMode.REPLICATED)
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
index 471b791bf5dfb..2dbc3aa1603bf 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
@@ -216,4 +216,5 @@ public void testUuidHandling() {
 
         assertEquals(1, (int)p.get(id));
     }
+
 }

From f69532d36a01b2f6665e94be085a46d1a22b664b Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Tue, 4 Apr 2017 11:03:40 +0300
Subject: [PATCH 415/516] GG-12957: Backport of IGNITE-4625 .NET: Fix java-only
 node stop

Solves "Ignite instance with this name has already been started" test failures

(cherry picked from commit 7696dd2)
---
 .../platform/PlatformStopIgniteTask.java      | 11 ++++++-
 .../Compute/AbstractTaskTest.cs               | 28 ++---------------
 .../Compute/MixedClusterTest.cs               | 31 +++----------------
 .../DeploymentTest.cs                         | 10 ++++++
 4 files changed, 26 insertions(+), 54 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/platform/PlatformStopIgniteTask.java b/modules/core/src/test/java/org/apache/ignite/platform/PlatformStopIgniteTask.java
index 238f0584cb738..92e3bc5f1b480 100644
--- a/modules/core/src/test/java/org/apache/ignite/platform/PlatformStopIgniteTask.java
+++ b/modules/core/src/test/java/org/apache/ignite/platform/PlatformStopIgniteTask.java
@@ -38,7 +38,16 @@ public class PlatformStopIgniteTask extends ComputeTaskAdapter<String, Boolean>
     /** {@inheritDoc} */
     @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
         @Nullable String arg) throws IgniteException {
-        return Collections.singletonMap(new PlatformStopIgniteJob(arg), F.first(subgrid));
+        ClusterNode node = subgrid.get(0);
+
+        for (ClusterNode n : subgrid) {
+            if (n.isLocal()) {
+                node = n;
+                break;
+            }
+        }
+
+        return Collections.singletonMap(new PlatformStopIgniteJob(arg), node);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/AbstractTaskTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/AbstractTaskTest.cs
index 6bcd01042e598..65bb369f002d4 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/AbstractTaskTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/AbstractTaskTest.cs
@@ -129,32 +129,8 @@ public void BeforeTest()
         [TestFixtureTearDown]
         public void StopClient()
         {
-            if (Grid1 != null)
-                Ignition.Stop(Grid1.Name, true);
-
-            if (_fork)
-            {
-                if (_proc2 != null) {
-                    _proc2.Kill();
-
-                    _proc2.Join();
-                }
-
-                if (_proc3 != null)
-                {
-                    _proc3.Kill();
-
-                    _proc3.Join();
-                }
-            }
-            else
-            {
-                if (_grid2 != null)
-                    Ignition.Stop(_grid2.Name, true);
-
-                if (_grid3 != null)
-                    Ignition.Stop(_grid3.Name, true);
-            }
+            Ignition.StopAll(true);
+            IgniteProcess.KillAll();
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/MixedClusterTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/MixedClusterTest.cs
index ab4e6ab88201a..41817eb676561 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/MixedClusterTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/MixedClusterTest.cs
@@ -21,8 +21,6 @@ namespace Apache.Ignite.Core.Tests.Compute
     using System.Collections;
     using System.Linq;
     using Apache.Ignite.Core.Cache;
-    using Apache.Ignite.Core.Cache.Query;
-    using Apache.Ignite.Core.Common;
     using Apache.Ignite.Core.Compute;
     using NUnit.Framework;
 
@@ -31,8 +29,12 @@ namespace Apache.Ignite.Core.Tests.Compute
     /// </summary>
     public class MixedClusterTest
     {
+        /** */
         private IIgnite _ignite;
+        
+        /** */
         private string _javaNodeName;
+
         /** */
         private const string SpringConfig = @"Config\Compute\compute-grid1.xml";
 
@@ -88,18 +90,6 @@ public void TestJavaTask()
             Assert.AreEqual(2, res.Count);
         }
 
-        /// <summary>
-        /// Tests the scan query.
-        /// </summary>
-        [Test]
-        public void TestScanQuery()
-        {
-            var cache = GetCache();
-            
-            // Scan query does not work in the mixed cluster.
-            Assert.Throws<IgniteException>(() => cache.Query(new ScanQuery<int, int>(new ScanFilter())).GetAll());
-        }
-
         /// <summary>
         /// Tests the cache invoke.
         /// </summary>
@@ -153,19 +143,6 @@ public int Invoke()
             }
         }
 
-        /// <summary>
-        /// Test filter.
-        /// </summary>
-        [Serializable]
-        private class ScanFilter : ICacheEntryFilter<int, int>
-        {
-            /** <inheritdoc /> */
-            public bool Invoke(ICacheEntry<int, int> entry)
-            {
-                return entry.Key < 100;
-            }
-        }
-
         /// <summary>
         /// Test processor.
         /// </summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/DeploymentTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/DeploymentTest.cs
index ab5a1a6aea116..20a5ffdca1a88 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/DeploymentTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/DeploymentTest.cs
@@ -107,6 +107,16 @@ public void TestCustomDeployment()
             }
         }
 
+        /// <summary>
+        /// Fixture tear down.
+        /// </summary>
+        [TestFixtureTearDown]
+        public void TestFixtureTearDown()
+        {
+            Ignition.StopAll(true);
+            IgniteProcess.KillAll();
+        }
+
         /// <summary>
         /// Verifies that custom-deployed node has started.
         /// </summary>

From ad49c5e66ec9a15c3996cc137afc44cc626e9f27 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 31 Oct 2017 18:50:46 +0300
Subject: [PATCH 416/516] Fixed javadoc.

---
 .../main/java/org/apache/ignite/IgniteSystemProperties.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 4bae00e4b739c..16ae83a1ee6d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -604,7 +604,8 @@ public final class IgniteSystemProperties {
     /** Ignite marshaller cache reread pause. */
     public static final String IGNITE_MARSHALLER_CACHE_REREAD_PAUSE = "IGNITE_MARSHALLER_CACHE_REREAD_PAUSE";
 
-    /** Class name of the closure {@link IgniteClosure<String, Throwable>} that
+    /**
+     * Class name of the closure {@link IgniteClosure}<String, Throwable> that
      * will be invoked once per topology and validates cache. If it returns or throws exception,
      * that means cache is invalid.*/
     public static final String IGNITE_CACHE_VALIDATOR = "IGNITE_CACHE_VALIDATOR";

From b35ff01af99594f4fe397340f1e0561ea771264a Mon Sep 17 00:00:00 2001
From: Yakov Zhdanov <yzhdanov@gridgain.com>
Date: Tue, 11 Apr 2017 19:47:10 +0300
Subject: [PATCH 417/516] GG-12991: Backport of IGNITE-4828 Improve the
 distribution of keys within partitions.

(cherry picked from commit 55ab10e)
---
 .../RendezvousAffinityFunction.java           | 26 ++++++++--
 ...ityFunctionFastPowerOfTwoHashSelfTest.java | 50 +++++++++++++++++++
 ...sAffinityFunctionStandardHashSelfTest.java | 50 +++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite2.java     |  4 ++
 4 files changed, 126 insertions(+), 4 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionStandardHashSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
index 3d21dd5b154fe..54df331948165 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
@@ -101,6 +101,9 @@ public class RendezvousAffinityFunction implements AffinityFunction, Externaliza
     /** Number of partitions. */
     private int parts;
 
+    /** Mask to use in calculation when partitions count is power of 2. */
+    private transient int mask = -1;
+
     /** Exclude neighbors flag. */
     private boolean exclNeighbors;
 
@@ -188,7 +191,9 @@ private RendezvousAffinityFunction(boolean exclNeighbors, int parts,
         A.ensure(parts > 0, "parts > 0");
 
         this.exclNeighbors = exclNeighbors;
-        this.parts = parts;
+
+        setPartitions(parts);
+
         this.backupFilter = backupFilter;
 
         try {
@@ -216,14 +221,20 @@ public int getPartitions() {
     }
 
     /**
-     * Sets total number of partitions.
+     * Sets total number of partitions.If the number of partitions is a power of two,
+     * the PowerOfTwo hashing method will be used.  Otherwise the Standard hashing
+     * method will be applied.
      *
      * @param parts Total number of partitions.
      */
     public void setPartitions(int parts) {
-        A.ensure(parts <= CacheConfiguration.MAX_PARTITIONS_COUNT, "parts <= " + CacheConfiguration.MAX_PARTITIONS_COUNT);
+        A.ensure(parts <= CacheConfiguration.MAX_PARTITIONS_COUNT,
+            "parts <= " + CacheConfiguration.MAX_PARTITIONS_COUNT);
+        A.ensure(parts > 0, "parts > 0");
 
         this.parts = parts;
+
+        mask = (parts & (parts - 1)) == 0 ? parts - 1 : -1;
     }
 
     /**
@@ -490,6 +501,12 @@ else if (affinityBackupFilter == null && backupFilter == null)
             throw new IllegalArgumentException("Null key is passed for a partition calculation. " +
                 "Make sure that an affinity key that is used is initialized properly.");
 
+        if (mask >= 0) {
+            int h;
+
+            return ((h = key.hashCode()) ^ (h >>> 16)) & mask;
+        }
+
         return U.safeAbs(key.hashCode() % parts);
     }
 
@@ -536,7 +553,8 @@ else if (affinityBackupFilter == null && backupFilter == null)
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-        parts = in.readInt();
+        setPartitions(in.readInt());
+
         exclNeighbors = in.readBoolean();
         hashIdRslvr = (AffinityNodeHashResolver)in.readObject();
         backupFilter = (IgniteBiPredicate<ClusterNode, ClusterNode>)in.readObject();
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest.java
new file mode 100644
index 0000000000000..683ffa2542b86
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.affinity.rendezvous;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.affinity.AbstractAffinityFunctionSelfTest;
+import org.apache.ignite.cache.affinity.AffinityFunction;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Tests for {@link RendezvousAffinityFunction}.
+ */
+public class RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest extends AbstractAffinityFunctionSelfTest {
+    /** Ignite. */
+    private static Ignite ignite;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        ignite = startGrid();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected AffinityFunction affinityFunction() {
+        AffinityFunction aff = new RendezvousAffinityFunction(512, null);
+
+        GridTestUtils.setFieldValue(aff, "ignite", ignite);
+
+        return aff;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionStandardHashSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionStandardHashSelfTest.java
new file mode 100644
index 0000000000000..ed47c57eb545d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionStandardHashSelfTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.affinity.rendezvous;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.affinity.AbstractAffinityFunctionSelfTest;
+import org.apache.ignite.cache.affinity.AffinityFunction;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Tests for {@link RendezvousAffinityFunction}.
+ */
+public class RendezvousAffinityFunctionStandardHashSelfTest extends AbstractAffinityFunctionSelfTest {
+    /** Ignite. */
+    private static Ignite ignite;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        ignite = startGrid();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected AffinityFunction affinityFunction() {
+        AffinityFunction aff = new RendezvousAffinityFunction(513, null);
+
+        GridTestUtils.setFieldValue(aff, "ignite", ignite);
+
+        return aff;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
index 6d0745dea8a74..5304cd4bd92bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
@@ -22,6 +22,8 @@
 import org.apache.ignite.cache.affinity.fair.FairAffinityFunctionExcludeNeighborsSelfTest;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionBackupFilterSelfTest;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionExcludeNeighborsSelfTest;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionStandardHashSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheConcurrentReadThroughTest;
 import org.apache.ignite.internal.processors.cache.CacheConfigurationLeakTest;
 import org.apache.ignite.internal.processors.cache.CacheDhtLocalPartitionAfterRemoveSelfTest;
@@ -182,6 +184,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(GridCacheAtomicNearReadersSelfTest.class));
         suite.addTest(new TestSuite(GridCachePartitionedAffinitySelfTest.class));
         suite.addTest(new TestSuite(RendezvousAffinityFunctionExcludeNeighborsSelfTest.class));
+        suite.addTest(new TestSuite(RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest.class));
+        suite.addTest(new TestSuite(RendezvousAffinityFunctionStandardHashSelfTest.class));
         suite.addTest(new TestSuite(FairAffinityFunctionExcludeNeighborsSelfTest.class));
         suite.addTest(new TestSuite(GridCacheRendezvousAffinityClientSelfTest.class));
         suite.addTest(new TestSuite(GridCachePartitionedProjectionAffinitySelfTest.class));

From f1236288e229e373c9b18191c75c6f1ac7346aab Mon Sep 17 00:00:00 2001
From: Alexey Popov <tank2.alex@gmail.com>
Date: Tue, 31 Oct 2017 14:33:58 +0300
Subject: [PATCH 418/516] Backported IGNITE-6690 DiscoverySpi: Clientmode
 Ignite should not fail on handshake errors

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  14 +-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  16 +-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |  22 ++
 .../tcp/TcpDiscoveryWithWrongServerTest.java  | 332 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   5 +-
 5 files changed, 383 insertions(+), 6 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 3657b514d2780..c7c2ffa73a1e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -702,11 +702,17 @@ else if (addrs.isEmpty()) {
                 }
 
                 if (X.hasCause(e, StreamCorruptedException.class)) {
-                    if (--sslConnectAttempts == 0)
-                        throw new IgniteSpiException("Unable to establish plain connection. " +
-                            "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+                    // StreamCorruptedException could be caused by remote node failover
+                    if (connectAttempts < 2) {
+                        connectAttempts++;
 
-                    continue;
+                        continue;
+                    }
+
+                    if (log.isDebugEnabled())
+                        log.debug("Connect failed with StreamCorruptedException, skip address: " + addr);
+
+                    break;
                 }
 
                 if (timeoutHelper.checkFailureTimeoutReached(e))
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 0f85e0612b942..a89eb08b2e2f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1250,7 +1250,7 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
 
                 errs.add(e);
 
-                if (X.hasCause(e, SSLException.class, StreamCorruptedException.class)) {
+                if (X.hasCause(e, SSLException.class)) {
                     if (--sslConnectAttempts == 0)
                         throw new IgniteException("Unable to establish secure connection. " +
                             "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
@@ -1258,6 +1258,20 @@ else if (U.currentTimeMillis() - noResStart > spi.joinTimeout)
                     continue;
                 }
 
+                if (X.hasCause(e, StreamCorruptedException.class)) {
+                    // StreamCorruptedException could be caused by remote node failover
+                    if (connectAttempts < 2) {
+                        connectAttempts++;
+
+                        continue;
+                    }
+
+                    if (log.isDebugEnabled())
+                        log.debug("Connect failed with StreamCorruptedException, skip address: " + addr);
+
+                    break;
+                }
+
                 if (timeoutHelper.checkFailureTimeoutReached(e))
                     break;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index ac423363184ea..5b94a8d403c5e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -23,6 +23,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Serializable;
+import java.io.StreamCorruptedException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.Socket;
@@ -40,7 +41,9 @@
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLServerSocketFactory;
 import javax.net.ssl.SSLSocketFactory;
 import org.apache.ignite.Ignite;
@@ -281,6 +284,9 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, T
     /** Maximum ack timeout value for receiving message acknowledgement in milliseconds (value is <tt>600,000ms</tt>). */
     public static final long DFLT_MAX_ACK_TIMEOUT = 10 * 60 * 1000;
 
+    /** Ssl message pattern for StreamCorruptedException. */
+    private static Pattern sslMsgPattern = Pattern.compile("invalid stream header: 150\\d0\\d00");
+
     /** Local address. */
     protected String locAddr;
 
@@ -1507,6 +1513,22 @@ protected <T> T readMessage(Socket sock, @Nullable InputStream in, long timeout)
                     "long GC pauses on remote node) [curTimeout=" + timeout +
                     ", rmtAddr=" + sock.getRemoteSocketAddress() + ", rmtPort=" + sock.getPort() + ']');
 
+            StreamCorruptedException streamCorruptedCause = X.cause(e, StreamCorruptedException.class);
+
+            if (streamCorruptedCause != null) {
+                // Lets check StreamCorruptedException for SSL Alert message
+                // Sample SSL Alert message: 15:03:03:00:02:02:0a
+                // 15 = Alert
+                // 03:03 = SSL version
+                // 00:02 = payload length
+                // 02:0a = critical (02) / unexpected message (0a)
+                // So, check message for "invalid stream header: 150X0X00"
+
+                String msg = streamCorruptedCause.getMessage();
+
+                if (msg != null && sslMsgPattern.matcher(msg).matches())
+                    streamCorruptedCause.initCause(new SSLException("Detected SSL alert in StreamCorruptedException"));
+            }
             throw e;
         }
         finally {
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
new file mode 100644
index 0000000000000..768f5f7f3d281
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestThread;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Client-based discovery SPI test with non-Ignite servers.
+ */
+public class TcpDiscoveryWithWrongServerTest extends GridCommonAbstractTest {
+    /** Non-Ignite Server port. */
+    private final static int SERVER_PORT = 47500;
+
+    /** Non-Ignite Server socket. */
+    private ServerSocket srvSock;
+
+    /** Count of accepted connections to non-Ignite Server. */
+    private int connCnt;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
+
+        ipFinder.setAddresses(Collections.singleton("127.0.0.1:" + Integer.toString(SERVER_PORT) + ".." +
+            Integer.toString(SERVER_PORT + 2)));
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpiWithOrderedIps().setIpFinder(ipFinder));
+
+        if (igniteInstanceName.startsWith("client"))
+            cfg.setClientMode(true);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopTcpThread();
+
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     * Starts tcp test thread
+     * @param workerFactory one of WorkerFactory
+     */
+    private void startTcpThread(final WorkerFactory workerFactory) {
+        connCnt = 0;
+
+        try {
+            srvSock = new ServerSocket(SERVER_PORT, 10, InetAddress.getByName("127.0.0.1"));
+        }
+        catch (Exception e) {
+            fail("Unexpected TcpServer exception " + e.getMessage());
+        }
+
+        new GridTestThread(new Runnable() {
+            @Override public void run() {
+                try {
+                    while(!Thread.currentThread().isInterrupted()) {
+                        Socket clientSock = srvSock.accept();
+
+                        connCnt++;
+
+                        // Create a new thread for socket connection.
+                        new GridTestThread(workerFactory.newWorker(clientSock)).start();
+                    }
+                }
+                catch (Exception e) {
+                    if (!srvSock.isClosed())
+                        e.printStackTrace();
+                }
+            }
+        }).start();
+    }
+
+    /**
+     * Stops tcp test thread
+     * @throws IOException IOException
+     */
+    private void stopTcpThread() throws IOException {
+        if (srvSock != null && !srvSock.isClosed())
+            srvSock.close();
+    }
+
+    /**
+     * Test that Client successfully ignores wrong responses during Discovery Handshake Procedure.
+     *
+     * @throws Exception in case of error.
+     */
+    public void testWrongHandshakeResponse() throws Exception {
+        startTcpThread(new SomeResponseWorker());
+
+        simpleTest();
+    }
+
+    /**
+     * Test that Client successfully ignores wrong responses during Discovery Handshake Procedure.
+     *
+     * @throws Exception in case of error.
+     */
+    public void testNoHandshakeResponse() throws Exception {
+        startTcpThread(new NoResponseWorker());
+
+        simpleTest();
+    }
+
+    /**
+     * Test that Client successfully ignores when server closes sockets after Discovery Handshake Request.
+     *
+     * @throws Exception in case of error.
+     */
+    public void testDisconnectOnRequest() throws Exception {
+        startTcpThread(new DisconnectOnRequestWorker());
+
+        simpleTest();
+    }
+
+    /**
+     * Test that Client successfully ignores when server closes sockets immediately.
+     *
+     * @throws Exception in case of error.
+     */
+    public void testEarlyDisconnect() throws Exception {
+        startTcpThread(new EarlyDisconnectWorker());
+
+        simpleTest();
+    }
+
+    /**
+     * Some simple sanity check with the Server and Client
+     * It is expected that both client and server could successfully perform Discovery Procedure when there is
+     * unknown (test) server in the ipFinder list.
+     */
+    private void simpleTest() {
+        try {
+            Ignite srv = startGrid("server");
+            Ignite client = startGrid("client");
+
+            awaitPartitionMapExchange();
+
+            assertEquals(2, srv.cluster().nodes().size());
+            assertEquals(2, client.cluster().nodes().size());
+            assertTrue(connCnt >= 2);
+
+            srv.getOrCreateCache(DEFAULT_CACHE_NAME).put(1, 1);
+
+            assertEquals(1, client.getOrCreateCache(DEFAULT_CACHE_NAME).get(1));
+        }
+        catch (Exception e) {
+            fail("Failed with unexpected exception: " + e.getMessage());
+        }
+    }
+
+    /**
+     * Just a factory for runnable workers
+     */
+    private interface WorkerFactory {
+        /**
+         * Creates a new worker for socket
+         * @param clientSock socket for worker
+         * @return runnable Worker
+         */
+        Runnable newWorker(Socket clientSock);
+    }
+
+    /**
+     * SocketWorker
+     */
+    private abstract class SocketWorker implements Runnable {
+        /** Client socket. */
+        Socket clientSock;
+
+        /**
+         * @param clientSock Client socket.
+         */
+        SocketWorker(Socket clientSock) {
+            this.clientSock = clientSock;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            try {
+                InputStream input = clientSock.getInputStream();
+                OutputStream output = clientSock.getOutputStream();
+                byte[] buf = new byte[1024];
+
+                while (!clientSock.isClosed() && input.read(buf) > 0)
+                    action(input, output);
+
+                if (!clientSock.isClosed())
+                    clientSock.close();
+            }
+            catch (IOException e) {
+                log.error("Unexpected error", e);
+            }
+        }
+
+        /**
+         * @param input socket input stream
+         * @param output socket output stream
+         * @throws IOException IOException
+         */
+        public abstract void action(InputStream input, OutputStream output) throws IOException;
+    }
+
+    /**
+     * SomeResponseWorker.
+     */
+    private class SomeResponseWorker implements WorkerFactory {
+        /** {@inheritDoc} */
+        @Override public Runnable newWorker(Socket clientSock) {
+            return new SocketWorker(clientSock) {
+                @Override public void action(InputStream input, OutputStream output) throws IOException {
+                    output.write("Some response".getBytes());
+
+                    log.error("TEST: Some response was sent to " + clientSock.getRemoteSocketAddress());
+                }
+            };
+        }
+    }
+
+    /**
+     * NoResponseWorker.
+     */
+    private class NoResponseWorker implements WorkerFactory {
+        /** {@inheritDoc} */
+        @Override public Runnable newWorker(Socket clientSock) {
+            return new SocketWorker(clientSock) {
+                @Override public void action(InputStream input, OutputStream output) throws IOException {
+                    log.error("TEST: No response was sent to " + clientSock.getRemoteSocketAddress());
+               }
+            };
+        }
+    }
+
+    /**
+     * DisconnectOnRequestWorker.
+     */
+    private class DisconnectOnRequestWorker implements WorkerFactory {
+        /** {@inheritDoc} */
+        @Override public Runnable newWorker(Socket clientSock) {
+            return new SocketWorker(clientSock) {
+                @Override public void action(InputStream input, OutputStream output) throws IOException {
+                    clientSock.close();
+
+                    log.error("TEST: Socket closed for " + clientSock.getRemoteSocketAddress());
+                }
+            };
+        }
+    }
+
+    /**
+     * EarlyDisconnectWorker.
+     */
+    private class EarlyDisconnectWorker implements WorkerFactory {
+        /** {@inheritDoc} */
+        @Override public Runnable newWorker(Socket clientSock) {
+            return new SocketWorker(clientSock) {
+                @Override public void action(InputStream input, OutputStream output) throws IOException {
+                    // No-op
+                }
+
+                @Override public void run() {
+                    try {
+                        clientSock.close();
+
+                        log.error("TEST: Socket closed for " + clientSock.getRemoteSocketAddress());
+                    }
+                    catch (IOException e) {
+                        log.error("Unexpected error", e);
+                    }
+                }
+            };
+        }
+    }
+
+    /**
+     * TcpDiscoverySpi with non-shuffled resolved IP addresses. We should ensure that in this test non-Ignite server
+     * is the first element of the addresses list
+     */
+    class TcpDiscoverySpiWithOrderedIps extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected Collection<InetSocketAddress> resolvedAddresses() throws IgniteSpiException {
+            Collection<InetSocketAddress> shuffled = super.resolvedAddresses();
+            List<InetSocketAddress> res = new ArrayList<>(shuffled);
+
+            Collections.sort(res, new Comparator<InetSocketAddress>() {
+                @Override public int compare(InetSocketAddress o1, InetSocketAddress o2) {
+                    return o1.toString().compareTo(o2.toString());
+                }
+            });
+
+            return res;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index aa9ac93078947..43e9153cf3084 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -41,6 +41,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiStartStopSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSecuredUnsecuredTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryWithWrongServerTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.jdbc.TcpDiscoveryJdbcIpFinderSelfTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinderSelfTest;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.sharedfs.TcpDiscoverySharedFsIpFinderSelfTest;
@@ -93,7 +94,9 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
         suite.addTest(new TestSuite(AuthenticationRestartTest.class));
 
-        //Client connect
+        suite.addTest(new TestSuite(TcpDiscoveryWithWrongServerTest.class));
+
+        // Client connect.
         suite.addTest(new TestSuite(IgniteClientConnectTest.class));
         suite.addTest(new TestSuite(IgniteClientReconnectMassiveShutdownTest.class));
 

From 4d51f4716b5c206d1bc70511f8c3ec480a29008d Mon Sep 17 00:00:00 2001
From: Alexey Popov <tank2.alex@gmail.com>
Date: Thu, 2 Nov 2017 13:12:03 +0300
Subject: [PATCH 419/516] Fixed TcpDiscoveryWithWrongServerTest.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../tcp/TcpDiscoveryWithWrongServerTest.java  | 52 ++++++++++---------
 1 file changed, 28 insertions(+), 24 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
index 768f5f7f3d281..ffd0d030c643e 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
@@ -29,6 +29,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.spi.IgniteSpiException;
@@ -40,14 +41,17 @@
  * Client-based discovery SPI test with non-Ignite servers.
  */
 public class TcpDiscoveryWithWrongServerTest extends GridCommonAbstractTest {
-    /** Non-Ignite Server port. */
+    /** Non-Ignite Server port #1. */
     private final static int SERVER_PORT = 47500;
 
-    /** Non-Ignite Server socket. */
-    private ServerSocket srvSock;
+    /** Non-Ignite Server port #2. */
+    private final static int LAST_SERVER_PORT = SERVER_PORT + 5;
+
+    /** Non-Ignite Server sockets. */
+    private List<ServerSocket> srvSocks = new ArrayList<>();
 
     /** Count of accepted connections to non-Ignite Server. */
-    private int connCnt;
+    private AtomicInteger connCnt = new AtomicInteger(0);
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -56,7 +60,7 @@ public class TcpDiscoveryWithWrongServerTest extends GridCommonAbstractTest {
         TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
 
         ipFinder.setAddresses(Collections.singleton("127.0.0.1:" + Integer.toString(SERVER_PORT) + ".." +
-            Integer.toString(SERVER_PORT + 2)));
+            Integer.toString(LAST_SERVER_PORT)));
 
         cfg.setDiscoverySpi(new TcpDiscoverySpiWithOrderedIps().setIpFinder(ipFinder));
 
@@ -68,7 +72,7 @@ public class TcpDiscoveryWithWrongServerTest extends GridCommonAbstractTest {
 
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
-        stopTcpThread();
+        stopTcpThreads();
 
         stopAllGrids();
 
@@ -79,15 +83,10 @@ public class TcpDiscoveryWithWrongServerTest extends GridCommonAbstractTest {
      * Starts tcp test thread
      * @param workerFactory one of WorkerFactory
      */
-    private void startTcpThread(final WorkerFactory workerFactory) {
-        connCnt = 0;
+    private void startTcpThread(final WorkerFactory workerFactory, final int port) throws Exception {
+        final ServerSocket srvSock = new ServerSocket(port, 10, InetAddress.getByName("127.0.0.1"));
 
-        try {
-            srvSock = new ServerSocket(SERVER_PORT, 10, InetAddress.getByName("127.0.0.1"));
-        }
-        catch (Exception e) {
-            fail("Unexpected TcpServer exception " + e.getMessage());
-        }
+        srvSocks.add(srvSock);
 
         new GridTestThread(new Runnable() {
             @Override public void run() {
@@ -95,7 +94,7 @@ private void startTcpThread(final WorkerFactory workerFactory) {
                     while(!Thread.currentThread().isInterrupted()) {
                         Socket clientSock = srvSock.accept();
 
-                        connCnt++;
+                        connCnt.getAndIncrement();
 
                         // Create a new thread for socket connection.
                         new GridTestThread(workerFactory.newWorker(clientSock)).start();
@@ -103,7 +102,7 @@ private void startTcpThread(final WorkerFactory workerFactory) {
                 }
                 catch (Exception e) {
                     if (!srvSock.isClosed())
-                        e.printStackTrace();
+                        log.error("Unexpected error", e);
                 }
             }
         }).start();
@@ -113,9 +112,10 @@ private void startTcpThread(final WorkerFactory workerFactory) {
      * Stops tcp test thread
      * @throws IOException IOException
      */
-    private void stopTcpThread() throws IOException {
-        if (srvSock != null && !srvSock.isClosed())
-            srvSock.close();
+    private void stopTcpThreads() throws IOException {
+        for (ServerSocket srvSock: srvSocks)
+            if (!srvSock.isClosed())
+                srvSock.close();
     }
 
     /**
@@ -124,7 +124,8 @@ private void stopTcpThread() throws IOException {
      * @throws Exception in case of error.
      */
     public void testWrongHandshakeResponse() throws Exception {
-        startTcpThread(new SomeResponseWorker());
+        startTcpThread(new SomeResponseWorker(), SERVER_PORT);
+        startTcpThread(new SomeResponseWorker(), LAST_SERVER_PORT);
 
         simpleTest();
     }
@@ -135,7 +136,8 @@ public void testWrongHandshakeResponse() throws Exception {
      * @throws Exception in case of error.
      */
     public void testNoHandshakeResponse() throws Exception {
-        startTcpThread(new NoResponseWorker());
+        startTcpThread(new NoResponseWorker(), SERVER_PORT);
+        startTcpThread(new NoResponseWorker(), LAST_SERVER_PORT);
 
         simpleTest();
     }
@@ -146,7 +148,8 @@ public void testNoHandshakeResponse() throws Exception {
      * @throws Exception in case of error.
      */
     public void testDisconnectOnRequest() throws Exception {
-        startTcpThread(new DisconnectOnRequestWorker());
+        startTcpThread(new DisconnectOnRequestWorker(), SERVER_PORT);
+        startTcpThread(new DisconnectOnRequestWorker(), LAST_SERVER_PORT);
 
         simpleTest();
     }
@@ -157,7 +160,8 @@ public void testDisconnectOnRequest() throws Exception {
      * @throws Exception in case of error.
      */
     public void testEarlyDisconnect() throws Exception {
-        startTcpThread(new EarlyDisconnectWorker());
+        startTcpThread(new EarlyDisconnectWorker(), SERVER_PORT);
+        startTcpThread(new EarlyDisconnectWorker(), LAST_SERVER_PORT);
 
         simpleTest();
     }
@@ -176,7 +180,7 @@ private void simpleTest() {
 
             assertEquals(2, srv.cluster().nodes().size());
             assertEquals(2, client.cluster().nodes().size());
-            assertTrue(connCnt >= 2);
+            assertTrue(connCnt.get() >= 2);
 
             srv.getOrCreateCache(DEFAULT_CACHE_NAME).put(1, 1);
 

From b083133a4710441ad25a434234fa0419dc084a23 Mon Sep 17 00:00:00 2001
From: apopov <tank2.alex@gmail.com>
Date: Fri, 3 Nov 2017 09:43:48 +0300
Subject: [PATCH 420/516] Backported IGNITE-6690 Fixed build error

---
 .../spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
index ffd0d030c643e..77db8711993d3 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
@@ -41,6 +41,9 @@
  * Client-based discovery SPI test with non-Ignite servers.
  */
 public class TcpDiscoveryWithWrongServerTest extends GridCommonAbstractTest {
+    /** */
+    public static final String DEFAULT_CACHE_NAME = "myCache";
+
     /** Non-Ignite Server port #1. */
     private final static int SERVER_PORT = 47500;
 

From 0677aaf8763af75325306390c0e81f8406f72b89 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Wed, 8 Nov 2017 17:17:56 +0300
Subject: [PATCH 421/516] GG-12992 Moved queries execution to sys-pool.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../cache/query/GridCacheDistributedQueryManager.java  |  2 +-
 .../processors/cache/query/GridCacheQueryRequest.java  | 10 +++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
index 8457180e7b3a7..b97000693b6f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
@@ -283,7 +283,7 @@ protected void removeQueryFuture(long reqId) {
                 false,
                 null,
                 req.keyValueFilter(),
-                req.partition() == -1 ? null : req.partition(),
+                req.partitionId() == -1 ? null : req.partitionId(),
                 req.className(),
                 req.clause(),
                 req.includeMetaData(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
index 3baafec54ebf0..4a7e14f9f8789 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
@@ -477,10 +477,18 @@ public int taskHash() {
         return taskHash;
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return Integer.MIN_VALUE;
+    }
+
     /**
+     * This method needed for scan query. {@link #GridCacheQueryRequest#partition()} method related with IO policy and cannot be used
+     * for this role. Otherwise Query will be processed in striped pool.
+     *
      * @return partition.
      */
-    public int partition() {
+    public int partitionId() {
         return part;
     }
 

From eb73b9df193d1724f45739759d5f4ac545401d1d Mon Sep 17 00:00:00 2001
From: apopov <tank2.alex@gmail.com>
Date: Fri, 27 Oct 2017 14:53:31 +0300
Subject: [PATCH 422/516] Backport IGNITE-4477 into 1.8.x

---
 .../cache/IgniteCacheFutureImpl.java          |   8 +-
 .../util/future/AsyncFutureListener.java      |  57 +++
 .../util/future/IgniteFutureImpl.java         |  33 +-
 .../org/apache/ignite/lang/IgniteFuture.java  |  24 +-
 .../future/IgniteCacheFutureImplTest.java     |  46 ++
 .../util/future/IgniteFutureImplTest.java     | 474 ++++++++++++++++--
 .../testsuites/IgniteLangSelfTestSuite.java   |   3 +
 .../schedule/ScheduleFutureImpl.java          |  49 +-
 .../schedule/GridScheduleSelfTest.java        |  84 +++-
 9 files changed, 723 insertions(+), 55 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/future/AsyncFutureListener.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/future/IgniteCacheFutureImplTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheFutureImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheFutureImpl.java
index 74cccc14cdaa8..c861be8894675 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheFutureImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheFutureImpl.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.concurrent.Executor;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteFutureCancelledCheckedException;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
@@ -43,7 +44,12 @@ public IgniteCacheFutureImpl(IgniteInternalFuture<V> fut) {
 
     /** {@inheritDoc} */
     @Override public <T> IgniteFuture<T> chain(IgniteClosure<? super IgniteFuture<V>, T> doneCb) {
-        return new IgniteCacheFutureImpl<>(chainInternal(doneCb));
+        return new IgniteCacheFutureImpl<>(chainInternal(doneCb, null));
+    }
+
+    /** {@inheritDoc} */
+    @Override public <T> IgniteFuture<T> chainAsync(IgniteClosure<? super IgniteFuture<V>, T> doneCb, Executor exec) {
+        return new IgniteCacheFutureImpl<>(chainInternal(doneCb, exec));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/AsyncFutureListener.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/AsyncFutureListener.java
new file mode 100644
index 0000000000000..460ce8bca1076
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/AsyncFutureListener.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.future;
+
+import java.util.concurrent.Executor;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+
+/**
+ * Wraps listener and executes it in specified executor.
+ */
+public class AsyncFutureListener<V> implements IgniteInClosure<IgniteFuture<V>> {
+    /** Serial version uid. */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final IgniteInClosure<? super IgniteFuture<V>> lsnr;
+
+    /** */
+    private final Executor exec;
+
+    /**
+     * @param lsnr Listener to be called asynchronously.
+     * @param exec Executor to process listener.
+     */
+    public AsyncFutureListener(IgniteInClosure<? super IgniteFuture<V>> lsnr, Executor exec) {
+        assert lsnr != null;
+        assert exec != null;
+
+        this.lsnr = lsnr;
+        this.exec = exec;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void apply(final IgniteFuture<V> fut) {
+        exec.execute(new Runnable() {
+            @Override public void run() {
+                lsnr.apply(fut);
+            }
+        });
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/IgniteFutureImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/IgniteFutureImpl.java
index 7d741548f5ef7..5427ea7d06895 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/IgniteFutureImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/IgniteFutureImpl.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.util.future;
 
+import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -28,6 +29,7 @@
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Implementation of public API future.
@@ -79,17 +81,35 @@ public IgniteInternalFuture<V> internalFuture() {
         fut.listen(new InternalFutureListener(lsnr));
     }
 
+    /** {@inheritDoc} */
+    @Override public void listenAsync(IgniteInClosure<? super IgniteFuture<V>> lsnr, Executor exec) {
+        A.notNull(lsnr, "lsnr");
+        A.notNull(exec, "exec");
+
+        fut.listen(new InternalFutureListener(new AsyncFutureListener<>(lsnr, exec)));
+    }
+
     /** {@inheritDoc} */
     @Override public <T> IgniteFuture<T> chain(final IgniteClosure<? super IgniteFuture<V>, T> doneCb) {
-        return new IgniteFutureImpl<>(chainInternal(doneCb));
+        return new IgniteFutureImpl<>(chainInternal(doneCb, null));
+    }
+
+    /** {@inheritDoc} */
+    @Override public <T> IgniteFuture<T> chainAsync(final IgniteClosure<? super IgniteFuture<V>, T> doneCb,
+        Executor exec) {
+        A.notNull(doneCb, "doneCb");
+        A.notNull(exec, "exec");
+
+        return new IgniteFutureImpl<>(chainInternal(doneCb, exec));
     }
 
     /**
      * @param doneCb Done callback.
      * @return Internal future
      */
-    protected  <T> IgniteInternalFuture<T> chainInternal(final IgniteClosure<? super IgniteFuture<V>, T> doneCb) {
-        return fut.chain(new C1<IgniteInternalFuture<V>, T>() {
+    protected  <T> IgniteInternalFuture<T> chainInternal(final IgniteClosure<? super IgniteFuture<V>, T> doneCb,
+        @Nullable Executor exec) {
+        C1<IgniteInternalFuture<V>, T> clos = new C1<IgniteInternalFuture<V>, T>() {
             @Override public T apply(IgniteInternalFuture<V> fut) {
                 assert IgniteFutureImpl.this.fut == fut;
 
@@ -100,7 +120,12 @@ protected  <T> IgniteInternalFuture<T> chainInternal(final IgniteClosure<? super
                     throw new GridClosureException(e);
                 }
             }
-        });
+        };
+
+        if (exec != null)
+            return fut.chain(clos, exec);
+
+        return fut.chain(clos);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/lang/IgniteFuture.java b/modules/core/src/main/java/org/apache/ignite/lang/IgniteFuture.java
index df1ab884f9b21..eec2546814e6a 100644
--- a/modules/core/src/main/java/org/apache/ignite/lang/IgniteFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/lang/IgniteFuture.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.lang;
 
+import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteInterruptedException;
@@ -111,11 +112,22 @@ public interface IgniteFuture<V> {
 
     /**
      * Registers listener closure to be asynchronously notified whenever future completes.
+     * Closure will be processed in thread that completes this future or (if future already
+     * completed) immediately in current thread.
      *
-     * @param lsnr Listener closure to register. If not provided - this method is no-op.
+     * @param lsnr Listener closure to register. Cannot be {@code null}.
      */
     public void listen(IgniteInClosure<? super IgniteFuture<V>> lsnr);
 
+    /**
+     * Registers listener closure to be asynchronously notified whenever future completes.
+     * Closure will be processed in specified executor.
+     *
+     * @param lsnr Listener closure to register. Cannot be {@code null}.
+     * @param exec Executor to run listener. Cannot be {@code null}.
+     */
+    public void listenAsync(IgniteInClosure<? super IgniteFuture<V>> lsnr, Executor exec);
+
     /**
      * Make a chained future to convert result of this future (when complete) into a new format.
      * It is guaranteed that done callback will be called only ONCE.
@@ -124,4 +136,14 @@ public interface IgniteFuture<V> {
      * @return Chained future that finishes after this future completes and done callback is called.
      */
     public <T> IgniteFuture<T> chain(IgniteClosure<? super IgniteFuture<V>, T> doneCb);
+
+    /**
+     * Make a chained future to convert result of this future (when complete) into a new format.
+     * It is guaranteed that done callback will be called only ONCE.
+     *
+     * @param doneCb Done callback that is applied to this future when it finishes to produce chained future result.
+     * @param exec Executor to run done callback. Cannot be {@code null}.
+     * @return Chained future that finishes after this future completes and done callback is called.
+     */
+    public <T> IgniteFuture<T> chainAsync(IgniteClosure<? super IgniteFuture<V>, T> doneCb, Executor exec);
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/future/IgniteCacheFutureImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/future/IgniteCacheFutureImplTest.java
new file mode 100644
index 0000000000000..46f1706f21d39
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/future/IgniteCacheFutureImplTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.future;
+
+import javax.cache.CacheException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.IgniteCacheFutureImpl;
+
+/**
+ * Tests IgniteCacheFutureImpl.
+ */
+public class IgniteCacheFutureImplTest extends IgniteFutureImplTest {
+    /** {@inheritDoc} */
+    @Override protected <V> IgniteFutureImpl<V> createFuture(IgniteInternalFuture<V> fut) {
+        return new IgniteCacheFutureImpl<>(fut);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Class<? extends Exception> expectedException() {
+        return CacheException.class;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void assertExpectedException(Exception e, Exception exp) {
+        if (exp instanceof IgniteException)
+            assertEquals(exp, e.getCause().getCause());
+        else
+            assertEquals(exp, e.getCause());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/future/IgniteFutureImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/future/IgniteFutureImplTest.java
index c1cc51ec60fd9..f97f1bcbfd4e9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/future/IgniteFutureImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/future/IgniteFutureImplTest.java
@@ -18,10 +18,17 @@
 package org.apache.ignite.internal.util.future;
 
 import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -29,18 +36,47 @@
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
 
 /**
  *
  */
 public class IgniteFutureImplTest extends GridCommonAbstractTest {
+    /** Context thread name. */
+    private static final String CTX_THREAD_NAME = "test-async";
+
+    /** Custom thread name. */
+    private static final String CUSTOM_THREAD_NAME = "test-custom-async";
+
+    /** Test executor. */
+    private ExecutorService ctxExec;
+
+    /** Custom executor. */
+    private ExecutorService customExec;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor")
+    @Override protected void beforeTest() throws Exception {
+        ctxExec = createExecutor(CTX_THREAD_NAME);
+        customExec = createExecutor(CUSTOM_THREAD_NAME);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        U.shutdownNow(getClass(), ctxExec, log);
+        U.shutdownNow(getClass(), customExec, log);
+
+        ctxExec = null;
+        customExec = null;
+    }
+
     /**
      * @throws Exception If failed.
      */
     public void testFutureGet() throws Exception {
         GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
 
-        IgniteFutureImpl<String> fut = new IgniteFutureImpl<>(fut0);
+        IgniteFutureImpl<String> fut = createFuture(fut0);
 
         assertFalse(fut.isDone());
 
@@ -73,7 +109,7 @@ public void testFutureGet() throws Exception {
     public void testFutureException() throws Exception {
         GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
 
-        final IgniteFutureImpl<String> fut = new IgniteFutureImpl<>(fut0);
+        final IgniteFutureImpl<String> fut = createFuture(fut0);
 
         assertFalse(fut.isDone());
 
@@ -87,35 +123,27 @@ public void testFutureException() throws Exception {
 
         fut0.onDone(err0);
 
-        IgniteException err = (IgniteException)GridTestUtils.assertThrows(log, new Callable<Void>() {
+        Exception err = (Exception)GridTestUtils.assertThrows(log, new Callable<Void>() {
             @Override public Void call() throws Exception {
                 fut.get();
 
                 return null;
             }
-        }, IgniteException.class, "test error");
+        }, expectedException(), "test error");
 
-        assertEquals(err0, err.getCause());
+        assertExpectedException(err, err0);
 
         assertTrue(fut.isDone());
 
-        assertTrue(fut.duration() > 0);
-
-        long dur0 = fut.duration();
-
-        U.sleep(100);
-
-        assertEquals(dur0, fut.duration());
-
-        err = (IgniteException)GridTestUtils.assertThrows(log, new Callable<Void>() {
+        err = (Exception)GridTestUtils.assertThrows(log, new Callable<Void>() {
             @Override public Void call() throws Exception {
                 fut.get();
 
                 return null;
             }
-        }, IgniteException.class, null);
+        }, expectedException(), null);
 
-        assertEquals(err0, err.getCause());
+        assertExpectedException(err, err0);
     }
 
     /**
@@ -124,21 +152,21 @@ public void testFutureException() throws Exception {
     public void testFutureIgniteException() throws Exception {
         GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
 
-        final IgniteFutureImpl<String> fut = new IgniteFutureImpl<>(fut0);
+        final IgniteFutureImpl<String> fut = createFuture(fut0);
 
         IgniteException err0 = new IgniteException("test error");
 
         fut0.onDone(err0);
 
-        IgniteException err = (IgniteException)GridTestUtils.assertThrows(log, new Callable<Void>() {
+        Exception err = (Exception)GridTestUtils.assertThrows(log, new Callable<Void>() {
             @Override public Void call() throws Exception {
                 fut.get();
 
                 return null;
             }
-        }, IgniteException.class, "test error");
+        }, expectedException(), "test error");
 
-        assertEquals(err0, err);
+        assertExpectedException(err, err0);
     }
 
     /**
@@ -147,7 +175,7 @@ public void testFutureIgniteException() throws Exception {
     public void testListeners() throws Exception {
         GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
 
-        IgniteFutureImpl<String> fut = new IgniteFutureImpl<>(fut0);
+        IgniteFutureImpl<String> fut = createFuture(fut0);
 
         final AtomicInteger lsnr1Cnt = new AtomicInteger();
 
@@ -192,7 +220,7 @@ public void testListenersOnError() throws Exception {
         {
             GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
 
-            IgniteFutureImpl<String> fut = new IgniteFutureImpl<>(fut0);
+            IgniteFutureImpl<String> fut = createFuture(fut0);
 
             final IgniteException err0 = new IgniteException("test error");
 
@@ -205,8 +233,8 @@ public void testListenersOnError() throws Exception {
 
                         fail();
                     }
-                    catch (IgniteException err) {
-                        assertEquals(err0, err);
+                    catch (IgniteException | CacheException err) {
+                        assertExpectedException(err, err0);
 
                         passed.set(true);
                     }
@@ -223,7 +251,7 @@ public void testListenersOnError() throws Exception {
         {
             GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
 
-            IgniteFutureImpl<String> fut = new IgniteFutureImpl<>(fut0);
+            IgniteFutureImpl<String> fut = createFuture(fut0);
 
             final IgniteCheckedException err0 = new IgniteCheckedException("test error");
 
@@ -236,8 +264,8 @@ public void testListenersOnError() throws Exception {
 
                         fail();
                     }
-                    catch (IgniteException err) {
-                        assertEquals(err0, err.getCause());
+                    catch (IgniteException | CacheException err) {
+                        assertExpectedException(err, err0);
 
                         passed.set(true);
                     }
@@ -252,13 +280,133 @@ public void testListenersOnError() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAsyncListeners() throws Exception {
+        GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
+
+        IgniteFutureImpl<String> fut = createFuture(fut0);
+
+        final CountDownLatch latch1 = new CountDownLatch(1);
+
+        IgniteInClosure<? super IgniteFuture<String>> lsnr1 = createAsyncListener(latch1, CUSTOM_THREAD_NAME, null);
+
+        assertFalse(fut.isDone());
+
+        fut.listenAsync(lsnr1, customExec);
+
+        U.sleep(100);
+
+        assertEquals(1, latch1.getCount());
+
+        fut0.onDone("test");
+
+        assert latch1.await(1, TimeUnit.SECONDS) : latch1.getCount();
+
+        checkAsyncListener(fut);
+        checkAsyncListener(createFuture(new GridFinishedFuture<>("test")));
+    }
+
+    /**
+     * @param fut Future.
+     */
+    private void checkAsyncListener(IgniteFutureImpl<String> fut) throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        IgniteInClosure<? super IgniteFuture<String>> lsnr = createAsyncListener(latch, CUSTOM_THREAD_NAME, null);
+
+        fut.listenAsync(lsnr, customExec);
+
+        assert latch.await(1, TimeUnit.SECONDS) : latch.getCount();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAsyncListenersOnError() throws Exception {
+        checkAsyncListenerOnError(new IgniteException("Test exception"));
+        checkAsyncListenerOnError(new IgniteCheckedException("Test checked exception"));
+    }
+
+    /**
+     * @param err0 Test exception.
+     */
+    private void checkAsyncListenerOnError(Exception err0) throws InterruptedException {
+        GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
+
+        IgniteFutureImpl<String> fut = createFuture(fut0);
+
+        final CountDownLatch latch1 = new CountDownLatch(1);
+
+        IgniteInClosure<? super IgniteFuture<String>> lsnr1 = createAsyncListener(latch1, CUSTOM_THREAD_NAME, err0);
+
+        fut.listenAsync(lsnr1, customExec);
+
+        assertEquals(1, latch1.getCount());
+
+        fut0.onDone(err0);
+
+        assert latch1.await(1, TimeUnit.SECONDS);
+
+        checkAsyncListenerOnError(err0, fut);
+        checkAsyncListenerOnError(err0, createFuture(new GridFinishedFuture<String>(err0)));
+    }
+
+    /**
+     * @param err0 Err 0.
+     * @param fut Future.
+     */
+    private void checkAsyncListenerOnError(Exception err0, IgniteFutureImpl<String> fut) throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        IgniteInClosure<? super IgniteFuture<String>> lsnr = createAsyncListener(latch, CUSTOM_THREAD_NAME, err0);
+
+        fut.listenAsync(lsnr, customExec);
+
+        assert latch.await(1, TimeUnit.SECONDS);
+    }
+
+    /**
+     * @param latch Latch.
+     */
+    @NotNull private CI1<IgniteFuture<String>> createAsyncListener(
+        final CountDownLatch latch,
+        final String threadName,
+        final Exception err
+    ) {
+        return new CI1<IgniteFuture<String>>() {
+            @Override public void apply(IgniteFuture<String> fut) {
+                try {
+                    String tname = Thread.currentThread().getName();
+
+                    assert tname.contains(threadName) : tname;
+
+                    assertEquals("test", fut.get());
+
+                    if (err != null)
+                        fail();
+                }
+                catch (IgniteException | CacheException e) {
+                    if (err != null)
+                        assertExpectedException(e, err);
+                    else
+                        throw e;
+                }
+                finally {
+                    latch.countDown();
+                }
+            }
+        };
+    }
+
     /**
      * @throws Exception If failed.
      */
     public void testChain() throws Exception {
         GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
 
-        IgniteFutureImpl<String> fut = new IgniteFutureImpl<>(fut0);
+        IgniteFutureImpl<String> fut = createFuture(fut0);
 
         IgniteFuture<Integer> chained = fut.chain(new C1<IgniteFuture<String>, Integer>() {
             @Override public Integer apply(IgniteFuture<String> fut) {
@@ -312,7 +460,7 @@ public void testChainError() throws Exception {
         {
             GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
 
-            IgniteFutureImpl<String> fut = new IgniteFutureImpl<>(fut0);
+            IgniteFutureImpl<String> fut = createFuture(fut0);
 
             final IgniteException err0 = new IgniteException("test error");
 
@@ -327,8 +475,8 @@ public void testChainError() throws Exception {
 
                         return -1;
                     }
-                    catch (IgniteException err) {
-                        assertEquals(err0, err);
+                    catch (IgniteException | CacheException err) {
+                        assertExpectedException(err, err0);
 
                         chainedPassed.set(true);
 
@@ -346,8 +494,8 @@ public void testChainError() throws Exception {
 
                         fail();
                     }
-                    catch (IgniteException err) {
-                        assertEquals(err0, err);
+                    catch (IgniteException | CacheException err) {
+                        assertExpectedException(err, err0);
 
                         lsnrPassed.set(true);
                     }
@@ -367,8 +515,8 @@ public void testChainError() throws Exception {
 
                 fail();
             }
-            catch (IgniteException err) {
-                assertEquals(err0, err);
+            catch (IgniteException | CacheException err) {
+                assertExpectedException(err, err0);
             }
 
             try {
@@ -376,15 +524,15 @@ public void testChainError() throws Exception {
 
                 fail();
             }
-            catch (IgniteException err) {
-                assertEquals(err0, err);
+            catch (IgniteException | CacheException err) {
+                assertExpectedException(err, err0);
             }
         }
 
         {
             GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
 
-            IgniteFutureImpl<String> fut = new IgniteFutureImpl<>(fut0);
+            IgniteFutureImpl<String> fut = createFuture(fut0);
 
             final IgniteCheckedException err0 = new IgniteCheckedException("test error");
 
@@ -399,8 +547,8 @@ public void testChainError() throws Exception {
 
                         return -1;
                     }
-                    catch (IgniteException err) {
-                        assertEquals(err0, err.getCause());
+                    catch (IgniteException | CacheException err) {
+                        assertExpectedException(err, err0);
 
                         chainedPassed.set(true);
 
@@ -418,8 +566,8 @@ public void testChainError() throws Exception {
 
                         fail();
                     }
-                    catch (IgniteException err) {
-                        assertEquals(err0, err.getCause());
+                    catch (IgniteException | CacheException err) {
+                        assertExpectedException(err, err0);
 
                         lsnrPassed.set(true);
                     }
@@ -439,8 +587,8 @@ public void testChainError() throws Exception {
 
                 fail();
             }
-            catch (IgniteException err) {
-                assertEquals(err0, err.getCause());
+            catch (IgniteException | CacheException err) {
+                assertExpectedException(err, err0);
             }
 
             try {
@@ -448,9 +596,243 @@ public void testChainError() throws Exception {
 
                 fail();
             }
-            catch (IgniteException err) {
-                assertEquals(err0, err.getCause());
+            catch (IgniteException | CacheException err) {
+                assertExpectedException(err, err0);
             }
         }
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testChainAsync() throws Exception {
+        GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
+
+        IgniteFuture<String> fut = createFuture(fut0);
+
+        C1<IgniteFuture<String>, Integer> chainClos = new C1<IgniteFuture<String>, Integer>() {
+            @Override public Integer apply(IgniteFuture<String> fut) {
+                assertEquals(CUSTOM_THREAD_NAME, Thread.currentThread().getName());
+
+                return Integer.valueOf(fut.get());
+            }
+        };
+
+        IgniteFuture<Integer> chained1 = fut.chainAsync(chainClos, customExec);
+
+        assertFalse(chained1.isDone());
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        class TestClosure implements CI1<IgniteFuture<Integer>> {
+            private final CountDownLatch latch;
+
+            private TestClosure(CountDownLatch latch) {
+                this.latch = latch;
+            }
+
+            @Override public void apply(IgniteFuture<Integer> fut) {
+                assertEquals(CUSTOM_THREAD_NAME, Thread.currentThread().getName());
+                assertEquals(10, (int)fut.get());
+
+                latch.countDown();
+            }
+        }
+
+        chained1.listen(new TestClosure(latch));
+
+        fut0.onDone("10");
+
+        // Chained future will be completed asynchronously.
+        chained1.get(100, TimeUnit.MILLISECONDS);
+
+        assertTrue(chained1.isDone());
+
+        assertEquals(10, (int)chained1.get());
+
+        assert latch.await(100, TimeUnit.MILLISECONDS);
+
+        assertTrue(fut.isDone());
+
+        assertEquals("10", fut.get());
+
+        // Test finished future
+        GridFinishedFuture<String> ffut0 = new GridFinishedFuture<>("10");
+
+        CountDownLatch latch1 = new CountDownLatch(1);
+
+        IgniteFuture<Integer> chained2 = createFuture(ffut0).chainAsync(chainClos, customExec);
+
+        chained2.listen(new TestClosure(latch1));
+
+        chained2.get(100, TimeUnit.MILLISECONDS);
+
+        assertTrue(chained2.isDone());
+
+        assertEquals(10, (int)chained2.get());
+
+        assert latch1.await(100, TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testChainAsyncOnError() throws Exception {
+        checkChainedOnError(new IgniteException("Test exception"));
+        checkChainedOnError(new IgniteCheckedException("Test checked exception"));
+        checkChainedOnErrorFinishedFuture(new IgniteException("Test exception"));
+        checkChainedOnErrorFinishedFuture(new IgniteCheckedException("Test checked exception"));
+    }
+
+    /**
+     * @param err Exception.
+     * @throws Exception If failed.
+     */
+    private void checkChainedOnError(final Exception err) throws Exception {
+        GridFutureAdapter<String> fut0 = new GridFutureAdapter<>();
+
+        IgniteFutureImpl<String> fut = createFuture(fut0);
+
+        // Chain callback will be invoked in specific executor.
+        IgniteFuture<Integer> chained1 = fut.chainAsync(new C1<IgniteFuture<String>, Integer>() {
+            @Override public Integer apply(IgniteFuture<String> fut) {
+                assertEquals(CUSTOM_THREAD_NAME, Thread.currentThread().getName());
+
+                try {
+                    fut.get();
+
+                    fail();
+                }
+                catch (IgniteException | CacheException e) {
+                    assertExpectedException(e, err);
+
+                    throw e;
+                }
+
+                return -1;
+            }
+        }, customExec);
+
+        assertFalse(chained1.isDone());
+        assertFalse(fut.isDone());
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        chained1.listen(new CI1<IgniteFuture<Integer>>() {
+            @Override public void apply(IgniteFuture<Integer> fut) {
+                try {
+                    assertEquals(CUSTOM_THREAD_NAME, Thread.currentThread().getName());
+
+                    fut.get();
+
+                    fail();
+                }
+                catch (IgniteException | CacheException e) {
+                    assertExpectedException(e, err);
+                }
+                finally {
+                    latch.countDown();
+                }
+            }
+        });
+
+        fut0.onDone(err);
+
+        assertExceptionThrown(err, chained1);
+        assertExceptionThrown(err, fut);
+
+        assertTrue(chained1.isDone());
+        assertTrue(fut.isDone());
+
+        assert latch.await(100, TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * @param err Err.
+     */
+    private void checkChainedOnErrorFinishedFuture(final Exception err) throws Exception {
+        IgniteFutureImpl<String> fut = createFuture(new GridFinishedFuture<String>(err));
+
+        // Chain callback will be invoked in specific executor.
+        IgniteFuture<Integer> chained1 = fut.chainAsync(new C1<IgniteFuture<String>, Integer>() {
+            @Override public Integer apply(IgniteFuture<String> fut) {
+                assertEquals(CUSTOM_THREAD_NAME, Thread.currentThread().getName());
+
+                try {
+                    fut.get();
+
+                    fail();
+                }
+                catch (IgniteException e) {
+                    assertExpectedException(e, err);
+
+                    throw e;
+                }
+
+                return -1;
+            }
+        }, customExec);
+
+
+        assertExceptionThrown(err, chained1);
+        assertExceptionThrown(err, fut);
+
+        assertTrue(chained1.isDone());
+        assertTrue(fut.isDone());
+    }
+
+    /**
+     * @param err Expected exception.
+     * @param fut Future.
+     */
+    private void assertExceptionThrown(Exception err, IgniteFuture<?> fut) {
+        try {
+            fut.get();
+
+            fail();
+        }
+        catch (IgniteException | CacheException e) {
+            assertExpectedException(e, err);
+        }
+    }
+
+    /**
+     * @param e Actual exception.
+     * @param exp Expected exception.
+     */
+    protected void assertExpectedException(Exception e, Exception exp) {
+        if (exp instanceof IgniteException)
+            assertEquals(exp, e);
+        else
+            assertEquals(exp, e.getCause());
+    }
+
+    /**
+     * @param name Name.
+     */
+    @NotNull private ExecutorService createExecutor(final String name) {
+        return Executors.newSingleThreadExecutor(new ThreadFactory() {
+            @Override public Thread newThread(@NotNull Runnable r) {
+                Thread t = new Thread(r);
+
+                t.setName(name);
+
+                return t;
+            }
+        });
+    }
+
+    /**
+     * @param fut Future.
+     */
+    protected <V> IgniteFutureImpl<V> createFuture(IgniteInternalFuture<V> fut) {
+        return new IgniteFutureImpl<>(fut);
+    }
+
+    /**
+     *
+     */
+    protected Class<? extends Exception> expectedException() {
+        return IgniteException.class;
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLangSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLangSelfTestSuite.java
index cfec1ec417fb1..70aec7270d06d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLangSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLangSelfTestSuite.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.util.future.GridCompoundFutureSelfTest;
 import org.apache.ignite.internal.util.future.GridEmbeddedFutureSelfTest;
 import org.apache.ignite.internal.util.future.GridFutureAdapterSelfTest;
+import org.apache.ignite.internal.util.future.IgniteCacheFutureImplTest;
 import org.apache.ignite.internal.util.future.IgniteFutureImplTest;
 import org.apache.ignite.internal.util.future.nio.GridNioEmbeddedFutureSelfTest;
 import org.apache.ignite.internal.util.future.nio.GridNioFutureSelfTest;
@@ -80,7 +81,9 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(GridEmbeddedFutureSelfTest.class));
         suite.addTest(new TestSuite(GridNioFutureSelfTest.class));
         suite.addTest(new TestSuite(GridNioEmbeddedFutureSelfTest.class));
+
         suite.addTest(new TestSuite(IgniteFutureImplTest.class));
+        suite.addTest(new TestSuite(IgniteCacheFutureImplTest.class));
 
         // Consistent hash tests.
         suite.addTest(new TestSuite(GridConsistentHashSelfTest.class));
diff --git a/modules/schedule/src/main/java/org/apache/ignite/internal/processors/schedule/ScheduleFutureImpl.java b/modules/schedule/src/main/java/org/apache/ignite/internal/processors/schedule/ScheduleFutureImpl.java
index ac23a7952d077..3536229dc9f49 100644
--- a/modules/schedule/src/main/java/org/apache/ignite/internal/processors/schedule/ScheduleFutureImpl.java
+++ b/modules/schedule/src/main/java/org/apache/ignite/internal/processors/schedule/ScheduleFutureImpl.java
@@ -25,6 +25,7 @@
 import java.util.Collection;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Matcher;
@@ -35,6 +36,7 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
+import org.apache.ignite.internal.util.future.AsyncFutureListener;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.future.IgniteFutureImpl;
 import org.apache.ignite.internal.util.lang.GridClosureException;
@@ -588,16 +590,43 @@ private void parsePatternParameters() throws IgniteCheckedException {
             notifyListener(lsnr, res, err);
     }
 
+    /** {@inheritDoc} */
+    @Override public void listenAsync(IgniteInClosure<? super IgniteFuture<R>> lsnr, Executor exec) {
+        A.notNull(lsnr, "lsnr");
+        A.notNull(exec, "exec");
+
+        listen(new AsyncFutureListener<>(lsnr, exec));
+    }
+
     /** {@inheritDoc} */
     @SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor")
     @Override public <T> IgniteFuture<T> chain(final IgniteClosure<? super IgniteFuture<R>, T> doneCb) {
+        A.notNull(doneCb, "doneCb");
+
+        return chain(doneCb, null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public <T> IgniteFuture<T> chainAsync(IgniteClosure<? super IgniteFuture<R>, T> doneCb, Executor exec) {
+        A.notNull(doneCb, "");
+        A.notNull(exec, "exec");
+
+        return chain(doneCb, exec);
+    }
+
+    /**
+     * @param doneCb Done callback.
+     * @param exec Executor.
+     * @return Chained future.
+     */
+    private <T> IgniteFuture<T> chain(final IgniteClosure<? super IgniteFuture<R>, T> doneCb, @Nullable Executor exec) {
         final GridFutureAdapter<T> fut = new GridFutureAdapter<T>() {
             @Override public String toString() {
                 return "ChainFuture[orig=" + ScheduleFutureImpl.this + ", doneCb=" + doneCb + ']';
             }
         };
 
-        listen(new CI1<IgniteFuture<R>>() {
+        IgniteInClosure<? super IgniteFuture<R>> lsnr = new CI1<IgniteFuture<R>>() {
             @Override public void apply(IgniteFuture<R> fut0) {
                 try {
                     fut.onDone(doneCb.apply(fut0));
@@ -617,7 +646,12 @@ private void parsePatternParameters() throws IgniteCheckedException {
                     throw e;
                 }
             }
-        });
+        };
+
+        if (exec != null)
+            lsnr = new AsyncFutureListener<>(lsnr, exec);
+
+        listen(lsnr);
 
         return new IgniteFutureImpl<>(fut);
     }
@@ -880,10 +914,21 @@ private static class ScheduleFutureSnapshot<R> implements SchedulerFuture<R> {
             ref.listen(lsnr);
         }
 
+        /** {@inheritDoc} */
+        @Override public void listenAsync(IgniteInClosure<? super IgniteFuture<R>> lsnr, Executor exec) {
+            ref.listenAsync(lsnr, exec);
+        }
+
         /** {@inheritDoc} */
         @Override public <T> IgniteFuture<T> chain(IgniteClosure<? super IgniteFuture<R>, T> doneCb) {
             return ref.chain(doneCb);
         }
+
+        /** {@inheritDoc} */
+        @Override public <T> IgniteFuture<T> chainAsync(IgniteClosure<? super IgniteFuture<R>, T> doneCb,
+            Executor exec) {
+            return ref.chainAsync(doneCb, exec);
+        }
     }
 
     /** {@inheritDoc} */
diff --git a/modules/schedule/src/test/java/org/apache/ignite/internal/processors/schedule/GridScheduleSelfTest.java b/modules/schedule/src/test/java/org/apache/ignite/internal/processors/schedule/GridScheduleSelfTest.java
index f0860f219f9ed..122d98ddcbe3a 100644
--- a/modules/schedule/src/test/java/org/apache/ignite/internal/processors/schedule/GridScheduleSelfTest.java
+++ b/modules/schedule/src/test/java/org/apache/ignite/internal/processors/schedule/GridScheduleSelfTest.java
@@ -19,21 +19,29 @@
 
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.lang.GridTuple;
 import org.apache.ignite.internal.util.typedef.CI1;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.scheduler.SchedulerFuture;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
 
 /**
  * Test for task scheduler.
@@ -43,9 +51,15 @@ public class GridScheduleSelfTest extends GridCommonAbstractTest {
     /** */
     private static final int NODES_CNT = 2;
 
+    /** Custom thread name. */
+    private static final String CUSTOM_THREAD_NAME = "custom-async-test";
+
     /** */
     private static AtomicInteger execCntr = new AtomicInteger(0);
 
+    /** Custom executor. */
+    private ExecutorService exec;
+
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         startGrids(NODES_CNT);
@@ -61,6 +75,22 @@ public class GridScheduleSelfTest extends GridCommonAbstractTest {
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         execCntr.set(0);
+        exec = Executors.newSingleThreadExecutor(new ThreadFactory() {
+            @Override public Thread newThread(@NotNull Runnable r) {
+                Thread t = new Thread(r);
+
+                t.setName(CUSTOM_THREAD_NAME);
+
+                return t;
+            }
+        });
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        U.shutdownNow(getClass(), exec, log);
+
+        exec = null;
     }
 
     /**
@@ -124,6 +154,54 @@ public void testScheduleRunnable() throws Exception {
                 }
             });
 
+            final SchedulerFuture<?> fut0 = fut;
+
+            //noinspection ThrowableNotThrown
+            assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    fut0.listenAsync(new IgniteInClosure<IgniteFuture<?>>() {
+                        @Override public void apply(IgniteFuture<?> fut) {
+                            // No-op
+                        }
+                    }, null);
+
+                    return null;
+                }
+            }, NullPointerException.class, null);
+
+            fut.listenAsync(new IgniteInClosure<IgniteFuture<?>>() {
+                @Override public void apply(IgniteFuture<?> fut) {
+                    assertEquals(Thread.currentThread().getName(), CUSTOM_THREAD_NAME);
+
+                    notifyCnt.incrementAndGet();
+                }
+            }, exec);
+
+            //noinspection ThrowableNotThrown
+            assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    fut0.chainAsync(new IgniteClosure<IgniteFuture<?>, String>() {
+                        @Override public String apply(IgniteFuture<?> fut) {
+                            // No-op
+
+                            return null;
+                        }
+                    }, null);
+
+                    return null;
+                }
+            }, NullPointerException.class, null);
+
+            IgniteFuture<String> chained1 = fut.chainAsync(new IgniteClosure<IgniteFuture<?>, String>() {
+                @Override public String apply(IgniteFuture<?> fut) {
+                    assertEquals(Thread.currentThread().getName(), CUSTOM_THREAD_NAME);
+
+                    fut.get();
+
+                    return "done-custom";
+                }
+            }, exec);
+
             long timeTillRun = freq + delay;
 
             info("Going to wait for the first run: " + timeTillRun);
@@ -135,6 +213,7 @@ public void testScheduleRunnable() throws Exception {
             assert !fut.isDone();
             assert !fut.isCancelled();
             assert fut.last() == null;
+            assertFalse(chained1.isDone());
 
             info("Going to wait for 2nd run: " + timeTillRun);
 
@@ -142,10 +221,13 @@ public void testScheduleRunnable() throws Exception {
             Thread.sleep(timeTillRun * 1000);
 
             assert fut.isDone();
-            assert notifyCnt.get() == 2;
+            assert notifyCnt.get() == 2 * 2;
             assert !fut.isCancelled();
             assert fut.last() == null;
 
+            assertEquals("done-custom", chained1.get());
+
+            assertTrue(chained1.isDone());
         }
         finally {
             assert fut != null;

From cae22a3324f90c739c1cdf51504188f164096396 Mon Sep 17 00:00:00 2001
From: apopov <tank2.alex@gmail.com>
Date: Thu, 9 Nov 2017 11:13:53 +0300
Subject: [PATCH 423/516] GG-13042 Backport GG-13035 changes to 1.9.x to keep
 libs in sync

---
 examples/pom-standalone-lgpl.xml |  8 +-------
 examples/pom-standalone.xml      |  8 +-------
 examples/pom.xml                 | 14 +-------------
 modules/clients/pom.xml          |  8 +-------
 modules/ignored-tests/pom.xml    | 10 ++--------
 5 files changed, 6 insertions(+), 42 deletions(-)

diff --git a/examples/pom-standalone-lgpl.xml b/examples/pom-standalone-lgpl.xml
index f664514d5c83f..e73c416a39e63 100644
--- a/examples/pom-standalone-lgpl.xml
+++ b/examples/pom-standalone-lgpl.xml
@@ -74,13 +74,7 @@
         <dependency>
             <groupId>com.google.code.simple-spring-memcached</groupId>
             <artifactId>spymemcached</artifactId>
-            <version>2.7.3</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-codec</groupId>
-                    <artifactId>commons-codec</artifactId>
-                </exclusion>
-            </exclusions>
+            <version>2.8.4</version>
         </dependency>
     </dependencies>
 
diff --git a/examples/pom-standalone.xml b/examples/pom-standalone.xml
index 029ca777ae242..aa93273234201 100644
--- a/examples/pom-standalone.xml
+++ b/examples/pom-standalone.xml
@@ -74,13 +74,7 @@
         <dependency>
             <groupId>com.google.code.simple-spring-memcached</groupId>
             <artifactId>spymemcached</artifactId>
-            <version>2.7.3</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-codec</groupId>
-                    <artifactId>commons-codec</artifactId>
-                </exclusion>
-            </exclusions>
+            <version>2.8.4</version>
         </dependency>
     </dependencies>
 
diff --git a/examples/pom.xml b/examples/pom.xml
index e7fd67d197dac..7bc96f570fb08 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -83,13 +83,7 @@
         <dependency>
             <groupId>com.google.code.simple-spring-memcached</groupId>
             <artifactId>spymemcached</artifactId>
-            <version>2.7.3</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-codec</groupId>
-                    <artifactId>commons-codec</artifactId>
-                </exclusion>
-            </exclusions>
+            <version>2.8.4</version>
         </dependency>
 
         <dependency>
@@ -152,12 +146,6 @@
                     <artifactId>ignite-spark</artifactId>
                     <version>${project.version}</version>
                 </dependency>
-
-                <dependency>
-                    <groupId>org.jboss.netty</groupId>
-                    <artifactId>netty</artifactId>
-                    <version>3.2.9.Final</version>
-                </dependency>
             </dependencies>
 
             <build>
diff --git a/modules/clients/pom.xml b/modules/clients/pom.xml
index 022edb78f3b66..e3804355f503f 100644
--- a/modules/clients/pom.xml
+++ b/modules/clients/pom.xml
@@ -44,14 +44,8 @@
         <dependency>
             <groupId>com.google.code.simple-spring-memcached</groupId>
             <artifactId>spymemcached</artifactId>
-            <version>2.7.3</version>
+            <version>2.8.4</version>
             <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-codec</groupId>
-                    <artifactId>commons-codec</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
 
         <dependency>
diff --git a/modules/ignored-tests/pom.xml b/modules/ignored-tests/pom.xml
index 00c7d554d7650..8e3993b29022f 100644
--- a/modules/ignored-tests/pom.xml
+++ b/modules/ignored-tests/pom.xml
@@ -164,14 +164,8 @@
         <dependency>
             <groupId>com.google.code.simple-spring-memcached</groupId>
             <artifactId>spymemcached</artifactId>
-            <version>2.7.3</version>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-codec</groupId>
-                    <artifactId>commons-codec</artifactId>
-                </exclusion>
-            </exclusions>
+            <version>2.8.4</version>
+            <scope>test</scope>
         </dependency>
 
         <dependency>

From 02b13a5d2f266bc28147b79e65ade7246a70f7d0 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Thu, 9 Nov 2017 18:10:31 +0300
Subject: [PATCH 424/516] ignite-6669 Do not call CacheStoreSessionListener if
 store operation is not executed

---
 .../store/GridCacheStoreManagerAdapter.java   |  98 +++++-
 ...tenerRWThroughDisabledAtomicCacheTest.java |  33 ++
 ...ThroughDisabledTransactionalCacheTest.java | 138 ++++++++
 ...nListenerReadWriteThroughDisabledTest.java | 296 +++++++++++++++++
 ...SessionListenerWriteBehindEnabledTest.java | 309 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   8 +-
 .../Cache/Store/CacheStoreSessionTest.cs      |  15 +-
 7 files changed, 869 insertions(+), 28 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledAtomicCacheTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledTransactionalCacheTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerReadWriteThroughDisabledTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index e8ce7236052f5..3dbec4d21a56a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -31,16 +31,12 @@
 import javax.cache.integration.CacheLoaderException;
 import javax.cache.integration.CacheWriterException;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.store.CacheStore;
-import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cache.store.CacheStoreSession;
 import org.apache.ignite.cache.store.CacheStoreSessionListener;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore;
 import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
@@ -112,6 +108,9 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
     /** */
     private boolean writeThrough;
 
+    /** */
+    private boolean readThrough;
+
     /** */
     private Collection<CacheStoreSessionListener> sesLsnrs;
 
@@ -129,6 +128,8 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
 
         writeThrough = cfg.isWriteThrough();
 
+        readThrough = cfg.isReadThrough();
+
         this.cfgStore = cfgStore;
 
         store = cacheStoreWrapper(ctx, cfgStore, cfg);
@@ -329,7 +330,7 @@ private CacheStore cacheStoreWrapper(GridKernalContext ctx,
                 log.debug(S.toString("Loading value from store for key",
                     "key", storeKey, true));
 
-            sessionInit0(tx);
+            sessionInit0(tx, StoreOperation.READ, false);
 
             boolean threwEx = true;
 
@@ -463,7 +464,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
             if (log.isDebugEnabled())
                 log.debug("Loading values from store for keys: " + keys0);
 
-            sessionInit0(tx);
+            sessionInit0(tx, StoreOperation.READ, false);
 
             boolean threwEx = true;
 
@@ -522,7 +523,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
             if (log.isDebugEnabled())
                 log.debug("Loading all values from store.");
 
-            sessionInit0(null);
+            sessionInit0(null, StoreOperation.READ, false);
 
             boolean threwEx = true;
 
@@ -588,7 +589,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
                     "val", val, true));
             }
 
-            sessionInit0(tx);
+            sessionInit0(tx, StoreOperation.WRITE, false);
 
             boolean threwEx = true;
 
@@ -640,7 +641,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
                 if (log.isDebugEnabled())
                     log.debug("Storing values in cache store [entries=" + entries + ']');
 
-                sessionInit0(tx);
+                sessionInit0(tx, StoreOperation.WRITE, false);
 
                 boolean threwEx = true;
 
@@ -693,7 +694,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
             if (log.isDebugEnabled())
                 log.debug(S.toString("Removing value from cache store", "key", key, true));
 
-            sessionInit0(tx);
+            sessionInit0(tx, StoreOperation.WRITE, false);
 
             boolean threwEx = true;
 
@@ -742,7 +743,7 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
                 log.debug(S.toString("Removing values from cache store",
                     "keys", keys0, true));
 
-            sessionInit0(tx);
+            sessionInit0(tx, StoreOperation.WRITE, false);
 
             boolean threwEx = true;
 
@@ -793,10 +794,10 @@ private void loadAllFromStore(@Nullable IgniteInternalTx tx,
         boolean storeSessionEnded) throws IgniteCheckedException {
         assert store != null;
 
-        sessionInit0(tx);
+        sessionInit0(tx, commit? StoreOperation.COMMIT: StoreOperation.ROLLBACK, false);
 
         try {
-            if (sesLsnrs != null) {
+            if (sesLsnrs != null && sesHolder.get().contains(store)) {
                 for (CacheStoreSessionListener lsnr : sesLsnrs)
                     lsnr.onSessionEnd(locSes, commit);
             }
@@ -835,7 +836,7 @@ private void handleClassCastException(ClassCastException e) throws IgniteChecked
 
     /** {@inheritDoc} */
     @Override public void writeBehindSessionInit() throws IgniteCheckedException {
-        sessionInit0(null);
+        sessionInit0(null, null, true);
     }
 
     /** {@inheritDoc} */
@@ -845,9 +846,12 @@ private void handleClassCastException(ClassCastException e) throws IgniteChecked
 
     /**
      * @param tx Current transaction.
+     * @param op Store operation.
+     * @param writeBehindStoreInitiator {@code true} if method call is initiated by {@link GridCacheWriteBehindStore}.
      * @throws IgniteCheckedException If failed.
      */
-    private void sessionInit0(@Nullable IgniteInternalTx tx) throws IgniteCheckedException {
+    private void sessionInit0(@Nullable IgniteInternalTx tx, @Nullable StoreOperation op,
+        boolean writeBehindStoreInitiator) throws IgniteCheckedException {
         assert sesHolder != null;
 
         SessionData ses;
@@ -869,8 +873,45 @@ private void sessionInit0(@Nullable IgniteInternalTx tx) throws IgniteCheckedExc
 
         sesHolder.set(ses);
 
+        notifyCacheStoreSessionListeners(ses, op, writeBehindStoreInitiator);
+    }
+
+    /**
+     * @param ses Current session.
+     * @param op Store operation.
+     * @param writeBehindStoreInitiator {@code True} if method call is initiated by {@link GridCacheWriteBehindStore}.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void notifyCacheStoreSessionListeners(SessionData ses, @Nullable StoreOperation op,
+        boolean writeBehindStoreInitiator) throws IgniteCheckedException {
         try {
-            if (!ses.started(store) && sesLsnrs != null) {
+            boolean notifyLsnrs = false;
+
+            if (writeBehindStoreInitiator)
+                notifyLsnrs = !ses.started(store) && sesLsnrs != null;
+            else {
+                assert op != null;
+
+                switch (op) {
+                    case READ:
+                        notifyLsnrs = readThrough && !ses.started(store) && sesLsnrs != null;
+                        break;
+
+                    case WRITE:
+                        notifyLsnrs = !cacheConfiguration().isWriteBehindEnabled() && writeThrough
+                            && !ses.started(store) && sesLsnrs != null;
+                        break;
+
+                    case COMMIT:
+                    case ROLLBACK:
+                        // No needs to start the session (if not started yet) and notify listeners.
+                        break;
+
+                    default:
+                        assert false : "Unexpected operation: " + op.toString();
+                }
+            }
+            if (notifyLsnrs) {
                 for (CacheStoreSessionListener lsnr : sesLsnrs)
                     lsnr.onSessionStart(locSes);
             }
@@ -886,7 +927,7 @@ private void sessionInit0(@Nullable IgniteInternalTx tx) throws IgniteCheckedExc
     private void sessionEnd0(@Nullable IgniteInternalTx tx, boolean threwEx) throws IgniteCheckedException {
         try {
             if (tx == null) {
-                if (sesLsnrs != null) {
+                if (sesLsnrs != null && sesHolder.get().contains(store)) {
                     for (CacheStoreSessionListener lsnr : sesLsnrs)
                         lsnr.onSessionEnd(locSes, !threwEx);
                 }
@@ -1010,6 +1051,14 @@ private boolean ended(CacheStore store) {
             return !started.remove(store);
         }
 
+        /**
+         * @param store Cache store.
+         * @return {@code True} if session started.
+         */
+        private boolean contains(CacheStore store) {
+            return started.contains(store);
+        }
+
         /** {@inheritDoc} */
         @Override public String toString() {
             return S.toString(SessionData.class, this, "tx", CU.txString(tx));
@@ -1312,4 +1361,19 @@ public String toString() {
             }
         }
     }
+
+    /** Enumeration that represents possible operations on the underlying store. */
+    private enum StoreOperation {
+        /** Read key-value pair from the underlying store. */
+        READ,
+
+        /** Update or remove key from the underlying store. */
+        WRITE,
+
+        /** Commit changes to the underlying store. */
+        COMMIT,
+
+        /** Rollback changes to the underlying store. */
+        ROLLBACK
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledAtomicCacheTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledAtomicCacheTest.java
new file mode 100644
index 0000000000000..6e28a52b30cc0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledAtomicCacheTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+
+/**
+ * This class tests that redundant calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+ * and {@link CacheStoreSessionListener#onSessionEnd(CacheStoreSession, boolean)} are not executed.
+ */
+public class CacheStoreListenerRWThroughDisabledAtomicCacheTest extends CacheStoreSessionListenerReadWriteThroughDisabledTest {
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return ATOMIC;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledTransactionalCacheTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledTransactionalCacheTest.java
new file mode 100644
index 0000000000000..fd784a364eca4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledTransactionalCacheTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store;
+
+import java.util.Random;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
+
+/**
+ * This class tests that redundant calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+ * and {@link CacheStoreSessionListener#onSessionEnd(CacheStoreSession, boolean)} are not executed.
+ */
+public class CacheStoreListenerRWThroughDisabledTransactionalCacheTest extends CacheStoreSessionListenerReadWriteThroughDisabledTest {
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return TRANSACTIONAL;
+    }
+
+    /**
+     * Tests {@link IgniteCache#get(Object)} with disabled read-through and write-through modes.
+     */
+    public void testTransactionalLookup() {
+        testTransactionalLookup(OPTIMISTIC, READ_COMMITTED);
+        testTransactionalLookup(OPTIMISTIC, REPEATABLE_READ);
+        testTransactionalLookup(OPTIMISTIC, SERIALIZABLE);
+
+        testTransactionalLookup(PESSIMISTIC, READ_COMMITTED);
+        testTransactionalLookup(PESSIMISTIC, REPEATABLE_READ);
+        testTransactionalLookup(PESSIMISTIC, SERIALIZABLE);
+    }
+
+    /**
+     * @param concurrency Transaction concurrency level.
+     * @param isolation Transaction isolation level.
+     */
+    private void testTransactionalLookup(TransactionConcurrency concurrency, TransactionIsolation isolation) {
+        IgniteCache cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        Random r = new Random();
+
+        try (Transaction tx = grid(0).transactions().txStart(concurrency, isolation)) {
+            for (int i = 0; i < CNT; ++i)
+                cache.get(r.nextInt());
+
+            tx.commit();
+        }
+    }
+
+    /**
+     * Tests {@link IgniteCache#put(Object, Object)} with disabled read-through and write-through modes.
+     */
+    public void testTransactionalUpdate() {
+        testTransactionalUpdate(OPTIMISTIC, READ_COMMITTED);
+        testTransactionalUpdate(OPTIMISTIC, REPEATABLE_READ);
+        testTransactionalUpdate(OPTIMISTIC, SERIALIZABLE);
+
+        testTransactionalUpdate(PESSIMISTIC, READ_COMMITTED);
+        testTransactionalUpdate(PESSIMISTIC, REPEATABLE_READ);
+        testTransactionalUpdate(PESSIMISTIC, SERIALIZABLE);
+    }
+
+    /**
+     * @param concurrency Transaction concurrency level.
+     * @param isolation Transaction isolation level.
+     */
+    private void testTransactionalUpdate(TransactionConcurrency concurrency, TransactionIsolation isolation) {
+        IgniteCache cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        Random r = new Random();
+
+        try (Transaction tx = grid(0).transactions().txStart(concurrency, isolation)) {
+            for (int i = 0; i < CNT; ++i)
+                cache.put(r.nextInt(), "test-value");
+
+            tx.commit();
+        }
+    }
+
+    /**
+     * Tests {@link IgniteCache#remove(Object)} with disabled read-through and write-through modes.
+     */
+    public void testTransactionalRemove() {
+        testTransactionalRemove(OPTIMISTIC, READ_COMMITTED);
+        testTransactionalRemove(OPTIMISTIC, REPEATABLE_READ);
+        testTransactionalRemove(OPTIMISTIC, SERIALIZABLE);
+
+        testTransactionalRemove(PESSIMISTIC, READ_COMMITTED);
+        testTransactionalRemove(PESSIMISTIC, REPEATABLE_READ);
+        testTransactionalRemove(PESSIMISTIC, SERIALIZABLE);
+    }
+
+    /**
+     * @param concurrency Transaction concurrency level.
+     * @param isolation Transaction isolation level.
+     */
+    private void testTransactionalRemove(TransactionConcurrency concurrency, TransactionIsolation isolation) {
+        IgniteCache cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        Random r = new Random();
+
+        try (Transaction tx = grid(0).transactions().txStart(concurrency, isolation)) {
+            for (int i = 0; i < CNT; ++i) {
+                int key = r.nextInt();
+
+                cache.put(key, "test-value");
+
+                cache.remove(key, "test-value");
+            }
+
+            tx.commit();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerReadWriteThroughDisabledTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerReadWriteThroughDisabledTest.java
new file mode 100644
index 0000000000000..22691213227a5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerReadWriteThroughDisabledTest.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store;
+
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.logging.Logger;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.configuration.FactoryBuilder;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import javax.sql.DataSource;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.store.jdbc.CacheJdbcStoreSessionListener;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
+
+/**
+ * This class tests that redundant calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+ * and {@link CacheStoreSessionListener#onSessionEnd(CacheStoreSession, boolean)} are not executed.
+ */
+public abstract class CacheStoreSessionListenerReadWriteThroughDisabledTest extends GridCacheAbstractSelfTest {
+    /** {@inheritDoc} */
+    protected int gridCount() {
+        return 2;
+    }
+
+    /** */
+    protected final int CNT = 100;
+
+    /** */
+    protected final String DEFAULT_CACHE_NAME = null;
+
+    /** {@inheritDoc} */
+    protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
+        CacheConfiguration cacheCfg = super.cacheConfiguration(igniteInstanceName);
+
+        cacheCfg.setName(DEFAULT_CACHE_NAME);
+
+        cacheCfg.setCacheStoreFactory(FactoryBuilder.factoryOf(EmptyCacheStore.class));
+
+        cacheCfg.setCacheStoreSessionListenerFactories(new CacheStoreSessionFactory());
+
+        cacheCfg.setReadThrough(false);
+        cacheCfg.setWriteThrough(false);
+
+        cacheCfg.setBackups(0);
+
+        return cacheCfg;
+    }
+
+    /** {@inheritDoc} */
+    protected NearCacheConfiguration nearConfiguration() {
+        return null;
+    }
+
+    /**
+     * Tests that there are no calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)} and
+     * {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+     * while {@link IgniteCache#get(Object)} performed.
+     *
+     * @throws Exception If failed.
+     */
+    public void testLookup() throws Exception {
+        IgniteCache<Object, Object> cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        Random r = new Random();
+
+        for (int i = 0; i < CNT; ++i)
+            cache.get(r.nextInt());
+    }
+
+    /**
+     * Tests that there are no calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)} and
+     * {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+     * while {@link IgniteCache#getAll(Set)} performed.
+     *
+     * @throws Exception If failed.
+     */
+    public void testBatchLookup() throws Exception {
+        IgniteCache<Object, Object> cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        Random r = new Random();
+
+        Set<Object> values = new HashSet<>();
+
+        for (int i = 0; i < CNT; ++i)
+            values.add(r.nextInt());
+
+        cache.getAll(values);
+    }
+
+    /**
+     * Tests that there are no calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)} and
+     * {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+     * while {@link IgniteCache#put(Object, Object)} performed.
+     *
+     * @throws Exception If failed.
+     */
+    public void testUpdate() throws Exception {
+        IgniteCache<Object, Object> cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        Random r = new Random();
+
+        for (int i = 0; i < CNT; ++i)
+            cache.put(r.nextInt(), "test-value");
+    }
+
+    /**
+     * Tests that there are no calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)} and
+     * {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+     * while {@link IgniteCache#putAll(Map)} performed.
+     *
+     * @throws Exception If failed.
+     */
+    public void testBatchUpdate() throws Exception {
+        IgniteCache<Object, Object> cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        Random r = new Random();
+
+        Map<Object, Object> values = new TreeMap<>();
+
+        for (int i = 0; i < CNT; ++i)
+            values.put(r.nextInt(), "test-value");
+
+        cache.putAll(values);
+    }
+
+    /**
+     * Tests that there are no calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)} and
+     * {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+     * while {@link IgniteCache#remove(Object)} performed.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRemove() throws Exception {
+        IgniteCache<Object, Object> cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        Random r = new Random();
+
+        for (int i = 0; i < CNT; ++i) {
+            int key = r.nextInt();
+
+            cache.put(key, "test-value");
+
+            cache.remove(key);
+        }
+    }
+
+    /**
+     * Tests that there are no calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)} and
+     * {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+     * while {@link IgniteCache#removeAll(Set)} performed.
+     *
+     * @throws Exception If failed.
+     */
+    public void testBatchRemove() throws Exception {
+        IgniteCache<Object, Object> cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        Random r = new Random();
+
+        Set<Object> values = new HashSet<>();
+
+        for (int i = 0; i < CNT; ++i) {
+            int key = r.nextInt();
+
+            cache.put(key, "test-value");
+
+            values.add(key);
+        }
+
+        cache.removeAll(values);
+    }
+
+    /**
+     * Cache store session factory.
+     */
+    public static class CacheStoreSessionFactory implements Factory<TestCacheStoreSessionListener> {
+        /** {@inheritDoc} */
+        @Override public TestCacheStoreSessionListener create() {
+            TestCacheStoreSessionListener lsnr = new TestCacheStoreSessionListener();
+            lsnr.setDataSource(new DataSourceStub());
+            return lsnr;
+        }
+    }
+
+    /**
+     * Test cache store session listener.
+     */
+    public static class TestCacheStoreSessionListener extends CacheJdbcStoreSessionListener {
+        /** {@inheritDoc} */
+        @Override public void onSessionStart(CacheStoreSession ses) {
+            fail("TestCacheStoreSessionListener.onSessionStart(CacheStoreSession) should not be called.");
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onSessionEnd(CacheStoreSession ses, boolean commit) {
+            fail("TestCacheStoreSessionListener.onSessionEnd(CacheStoreSession, boolean) should not be called.");
+        }
+    }
+
+    /** Empty cache store implementation. All overridden methods should not be called while the test is running. */
+    public static class EmptyCacheStore extends CacheStoreAdapter {
+        /** {@inheritDoc} */
+        @Override public Object load(Object key) throws CacheLoaderException {
+            fail("EmptyCacheStore.load(Object) should not be called.");
+
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry entry) throws CacheWriterException {
+            fail("EmptyCacheStore.write(Cache.Entry) should not be called.");
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            fail("EmptyCacheStore.delete(Object) should not be called.");
+        }
+    }
+
+    /**
+     * Data source stub which should not be called.
+     */
+    public static class DataSourceStub implements DataSource, Serializable {
+        /** {@inheritDoc} */
+        @Override public Connection getConnection() throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public Connection getConnection(String username, String password) throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public <T> T unwrap(Class<T> iface) throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isWrapperFor(Class<?> iface) throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public PrintWriter getLogWriter() throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void setLogWriter(PrintWriter out) throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void setLoginTimeout(int seconds) throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public int getLoginTimeout() throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException {
+            throw new UnsupportedOperationException();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java
new file mode 100644
index 0000000000000..2b04309a40422
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store;
+
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Logger;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.configuration.FactoryBuilder;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import javax.sql.DataSource;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.store.jdbc.CacheJdbcStoreSessionListener;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
+import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore;
+import org.apache.ignite.resources.IgniteInstanceResource;
+
+/**
+ * This class tests that calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+ * and {@link CacheStoreSessionListener#onSessionEnd(CacheStoreSession, boolean)} are executed from
+ * {@link GridCacheWriteBehindStore} only.
+ */
+public class CacheStoreSessionListenerWriteBehindEnabledTest extends GridCacheAbstractSelfTest {
+    /** */
+    protected final static int CNT = 100;
+
+    /** */
+    protected final String DEFAULT_CACHE_NAME = null;
+
+    /** */
+    private final static int WRITE_BEHIND_FLUSH_FREQUENCY = 1000;
+
+    /** */
+    private static final List<OperationType> operations = Collections.synchronizedList(new ArrayList<OperationType>());
+
+    /** */
+    private static final AtomicInteger entryCnt = new AtomicInteger();
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
+        CacheConfiguration cacheCfg = super.cacheConfiguration(igniteInstanceName);
+
+        cacheCfg.setName(DEFAULT_CACHE_NAME);
+
+        cacheCfg.setCacheStoreFactory(FactoryBuilder.factoryOf(EmptyCacheStore.class));
+
+        cacheCfg.setCacheStoreSessionListenerFactories(new CacheStoreSessionFactory());
+
+        cacheCfg.setReadThrough(true);
+        cacheCfg.setWriteThrough(true);
+
+        cacheCfg.setWriteBehindEnabled(true);
+        cacheCfg.setWriteBehindBatchSize(CNT * 2);
+        cacheCfg.setWriteBehindFlushFrequency(WRITE_BEHIND_FLUSH_FREQUENCY);
+
+        cacheCfg.setBackups(0);
+
+        return cacheCfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        operations.clear();
+
+        entryCnt.set(0);
+    }
+
+    /**
+     * Tests that there are no redundant calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+     * while {@link IgniteCache#get(Object)} performed.
+     */
+    public void testLookup() {
+        IgniteCache<Object, Object> cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < CNT; ++i)
+            cache.get(i);
+
+        checkSessionCounters(CNT);
+    }
+
+    /**
+     * Tests that there are no redundant calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+     * while {@link IgniteCache#put(Object, Object)} performed.
+     */
+    public void testUpdate() {
+        IgniteCache<Object, Object> cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < CNT; ++i)
+            cache.put(i, i);
+
+        checkSessionCounters(1);
+    }
+
+    /**
+     * Tests that there are no redundant calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
+     * while {@link IgniteCache#remove(Object)} performed.
+     */
+    public void testRemove() {
+        IgniteCache<Object, Object> cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < CNT; ++i) {
+            cache.remove(i);
+        }
+
+        checkSessionCounters(1);
+    }
+
+    /**
+     * @param startedSessions Number of expected sessions.
+     */
+    private void checkSessionCounters(int startedSessions) {
+        try {
+            // Wait for GridCacheWriteBehindStore
+            Thread.sleep(WRITE_BEHIND_FLUSH_FREQUENCY * 4);
+
+            assertEquals(CNT, entryCnt.get());
+
+            checkOpCount(operations, OperationType.SESSION_START, startedSessions);
+
+            checkOpCount(operations, OperationType.SESSION_END, startedSessions);
+        }
+        catch (InterruptedException e) {
+            throw new IgniteException("Failed to wait for the GridCacheWriteBehindStore due to interruption.", e);
+        }
+    }
+
+    /**
+     * @param operations List of {@link OperationType}.
+     * @param op Operation.
+     * @param expected Expected number of operations for the given {@code op}.
+     */
+    private void checkOpCount(List<OperationType> operations, OperationType op, int expected) {
+        int n = 0;
+
+        for (OperationType o : operations) {
+            if (op.equals(o))
+                ++n;
+        }
+
+        assertEquals("Operation=" + op.name(), expected, n);
+    }
+
+    /**
+     * Operation type.
+     */
+    public enum OperationType {
+        /**
+         * Cache store session started.
+         */
+        SESSION_START,
+
+        /**
+         * Cache store session ended.
+         */
+        SESSION_END,
+    }
+
+    /**
+     * Cache store session factory.
+     */
+    public static class CacheStoreSessionFactory implements Factory<TestCacheStoreSessionListener> {
+        /** {@inheritDoc} */
+        @Override public TestCacheStoreSessionListener create() {
+            TestCacheStoreSessionListener lsnr = new TestCacheStoreSessionListener();
+            lsnr.setDataSource(new DataSourceStub());
+            return lsnr;
+        }
+    }
+
+    /**
+     * Test cache store session listener.
+     */
+    public static class TestCacheStoreSessionListener extends CacheJdbcStoreSessionListener {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public void onSessionStart(CacheStoreSession ses) {
+            operations.add(OperationType.SESSION_START);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onSessionEnd(CacheStoreSession ses, boolean commit) {
+            operations.add(OperationType.SESSION_END);
+        }
+    }
+
+    /**
+     * Test cache store.
+     *
+     * {@link EmptyCacheStore#writeAll(Collection)} and {@link EmptyCacheStore#deleteAll(Collection)} should be called
+     * by {@link GridCacheWriteBehindStore}.
+     */
+    public static class EmptyCacheStore extends CacheStoreAdapter<Object, Object> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public Object load(Object key) throws CacheLoaderException {
+            entryCnt.getAndIncrement();
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void writeAll(Collection<Cache.Entry<?, ?>> entries) {
+            entryCnt.addAndGet(entries.size());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry entry) throws CacheWriterException {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void deleteAll(Collection<?> keys) {
+            entryCnt.addAndGet(keys.size());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+        }
+    }
+
+    /**
+     * Data source stub which should not be called.
+     */
+    public static class DataSourceStub implements DataSource, Serializable {
+        /** {@inheritDoc} */
+        @Override public Connection getConnection() throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public Connection getConnection(String username, String password) throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public <T> T unwrap(Class<T> iface) throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isWrapperFor(Class<?> iface) throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public PrintWriter getLogWriter() throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void setLogWriter(PrintWriter out) throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void setLoginTimeout(int seconds) throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public int getLoginTimeout() throws SQLException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException {
+            throw new UnsupportedOperationException();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index c5667ba9a97e8..94492fa9e0b2c 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -18,6 +18,9 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.cache.store.CacheStoreListenerRWThroughDisabledAtomicCacheTest;
+import org.apache.ignite.cache.store.CacheStoreListenerRWThroughDisabledTransactionalCacheTest;
+import org.apache.ignite.cache.store.CacheStoreSessionListenerWriteBehindEnabledTest;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcStoreSessionListenerSelfTest;
 import org.apache.ignite.internal.processors.GridCacheTxLoadFromStoreOnLockSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheClientStoreSelfTest;
@@ -283,6 +286,9 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheOffheapMapEntrySelfTest.class);
 
         suite.addTestSuite(CacheJdbcStoreSessionListenerSelfTest.class);
+        suite.addTestSuite(CacheStoreListenerRWThroughDisabledAtomicCacheTest.class);
+        suite.addTestSuite(CacheStoreListenerRWThroughDisabledTransactionalCacheTest.class);
+        suite.addTestSuite(CacheStoreSessionListenerWriteBehindEnabledTest.class);
 
         suite.addTestSuite(CacheClientStoreSelfTest.class);
         suite.addTestSuite(CacheStoreUsageMultinodeStaticStartAtomicTest.class);
@@ -350,4 +356,4 @@ public static TestSuite suite() throws Exception {
 
         return suite;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs
index 54e0414c23f47..8ab3b25c65b94 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs
@@ -74,7 +74,7 @@ public void AfterTests()
         {
             Ignition.StopAll(true);
         }
-        
+
         /// <summary>
         /// Test basic session API.
         /// </summary>
@@ -97,13 +97,8 @@ public void TestSession()
                 tx.Rollback();
             }
 
-            Assert.AreEqual(1, _dumps.Count);
-            var ops = _dumps.First();
-            Assert.AreEqual(1, ops.Count);
-
-            Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.SesEnd && !op.Commit));
-
-            _dumps = new ConcurrentBag<ICollection<Operation>>();
+            // SessionEnd should not be called.
+            Assert.AreEqual(0, _dumps.Count);
 
             // 2. Test puts.
             using (var tx = ignite.GetTransactions().TxStart())
@@ -209,7 +204,7 @@ private ICollection<Operation> GetOperations()
                 }
 
                 return (ICollection<Operation>) ops;
-            } 
+            }
         }
 
         /// <summary>
@@ -245,7 +240,7 @@ public Operation(string cacheName, OperationType type, int key, int val) : this(
             /// Cache name.
             /// </summary>
             public string CacheName { get; private set; }
-            
+
             /// <summary>
             /// Operation type.
             /// </summary>

From 5372606f28a97de169691ba34f354b106048b6b1 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 10 Nov 2017 17:33:52 +0300
Subject: [PATCH 425/516] Backport of IGNITE-6649: Added eviction policy
 factory to cache configuration.

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit 6579e69)
---
 .../org/apache/ignite/cache/CacheMode.java    |    2 +-
 .../AbstractEvictionPolicyFactory.java        |  104 ++
 .../fifo/FifoEvictionPolicyFactory.java       |   72 ++
 .../lru/LruEvictionPolicyFactory.java         |   72 ++
 .../sorted/SortedEvictionPolicyFactory.java   |   98 ++
 .../configuration/CacheConfiguration.java     |   42 +-
 .../configuration/NearCacheConfiguration.java |   41 +
 .../processors/cache/GridCacheAttributes.java |   19 +
 .../cache/GridCacheEvictionManager.java       |   15 +-
 .../processors/cache/GridCacheProcessor.java  |   31 +-
 .../processors/cache/GridCacheUtils.java      |    1 +
 .../processors/igfs/IgfsHelperImpl.java       |    8 +-
 .../internal/processors/igfs/IgfsImpl.java    |    4 +-
 ...CacheConfigurationConsistencySelfTest.java |   50 +
 .../GridCacheNearEvictionEventSelfTest.java   |    5 -
 .../EvictionPolicyFactoryAbstractTest.java    | 1074 +++++++++++++++++
 .../FifoEvictionPolicyFactorySelfTest.java    |  261 ++++
 .../lru/LruEvictionPolicyFactorySelfTest.java |  352 ++++++
 .../SortedEvictionPolicyFactorySelfTest.java  |  264 ++++
 .../IgniteCacheEvictionSelfTestSuite.java     |    6 +
 .../apache/ignite/yardstick/IgniteNode.java   |    6 +-
 21 files changed, 2506 insertions(+), 21 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicyFactory.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/cache/eviction/fifo/FifoEvictionPolicyFactory.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/cache/eviction/lru/LruEvictionPolicyFactory.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/cache/eviction/sorted/SortedEvictionPolicyFactory.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/EvictionPolicyFactoryAbstractTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/FifoEvictionPolicyFactorySelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruEvictionPolicyFactorySelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/sorted/SortedEvictionPolicyFactorySelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheMode.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheMode.java
index e75fa0c9c3b24..4171b1ac28074 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/CacheMode.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheMode.java
@@ -55,7 +55,7 @@ public enum CacheMode {
      * <p>
      * Note that partitioned cache is always fronted by local
      * {@code 'near'} cache which stores most recent data. You
-     * can configure the size of near cache via {@link NearCacheConfiguration#getNearEvictionPolicy()}
+     * can configure the size of near cache via {@link NearCacheConfiguration#getNearEvictionPolicyFactory()}
      * configuration property.
      */
     PARTITIONED;
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicyFactory.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicyFactory.java
new file mode 100644
index 0000000000000..012c7ee522ea9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicyFactory.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.eviction;
+
+import javax.cache.configuration.Factory;
+import org.apache.ignite.internal.util.typedef.internal.A;
+
+/**
+ * Common functionality implementation for eviction policies factories.
+ */
+public abstract class AbstractEvictionPolicyFactory<T> implements Factory<T> {
+    /** */
+    private int maxSize;
+
+    /** */
+    private int batchSize = 1;
+
+    /** */
+    private long maxMemSize;
+
+    /**
+     * Sets maximum allowed size of cache before entry will start getting evicted.
+     *
+     * @param max Maximum allowed size of cache before entry will start getting evicted.
+     * @return {@code this} for chaining.
+     */
+    public AbstractEvictionPolicyFactory setMaxSize(int max) {
+        A.ensure(max >= 0, "max >= 0");
+
+        this.maxSize = max;
+
+        return this;
+    }
+
+    /**
+     * Gets maximum allowed size of cache before entry will start getting evicted.
+     *
+     * @return Maximum allowed size of cache before entry will start getting evicted.
+     */
+    public int getMaxSize() {
+        return maxSize;
+    }
+
+    /**
+     * Sets batch size.
+     *
+     * @param batchSize Batch size.
+     * @return {@code this} for chaining.
+     */
+    public AbstractEvictionPolicyFactory setBatchSize(int batchSize) {
+        A.ensure(batchSize > 0, "batchSize > 0");
+
+        this.batchSize = batchSize;
+
+        return this;
+    }
+
+    /**
+     * Gets batch size.
+     *
+     * @return batch size.
+     */
+    public int getBatchSize() {
+        return batchSize;
+    }
+
+    /**
+     * Sets maximum allowed cache size in bytes.
+     *
+     * @return {@code this} for chaining.
+     */
+    public AbstractEvictionPolicyFactory setMaxMemorySize(long maxMemSize) {
+        A.ensure(maxMemSize >= 0, "maxMemSize >= 0");
+
+        this.maxMemSize = maxMemSize;
+
+        return this;
+    }
+
+    /**
+     * Gets maximum allowed cache size in bytes.
+     *
+     * @return maximum allowed cache size in bytes.
+     */
+    public long getMaxMemorySize() {
+        return maxMemSize;
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/fifo/FifoEvictionPolicyFactory.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/fifo/FifoEvictionPolicyFactory.java
new file mode 100644
index 0000000000000..856865af6aa3b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/fifo/FifoEvictionPolicyFactory.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.eviction.fifo;
+
+import org.apache.ignite.cache.eviction.AbstractEvictionPolicyFactory;
+
+/**
+ * Factory class for {@link FifoEvictionPolicy}.
+ *
+ * Creates cache Eviction policy based on {@code First In First Out (FIFO)} algorithm and supports batch eviction.
+ * <p>
+ * The eviction starts in the following cases:
+ * <ul>
+ *     <li>The cache size becomes {@code batchSize} elements greater than the maximum size.</li>
+ *     <li>
+ *         The size of cache entries in bytes becomes greater than the maximum memory size.
+ *         The size of cache entry calculates as sum of key size and value size.
+ *     </li>
+ * </ul>
+ * <b>Note:</b>Batch eviction is enabled only if maximum memory limit isn't set ({@code maxMemSize == 0}).
+ * {@code batchSize} elements will be evicted in this case. The default {@code batchSize} value is {@code 1}.
+ * <p>
+ * {@link FifoEvictionPolicy} implementation is very efficient since it does not create any additional
+ * table-like data structures. The {@code FIFO} ordering information is
+ * maintained by attaching ordering metadata to cache entries.
+ */
+public class FifoEvictionPolicyFactory<K, V> extends AbstractEvictionPolicyFactory<FifoEvictionPolicy<K, V>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Constructor. */
+    public FifoEvictionPolicyFactory() {
+    }
+
+    /** Constructor. */
+    public FifoEvictionPolicyFactory(int maxSize) {
+        setMaxSize(maxSize);
+    }
+
+    /** */
+    public FifoEvictionPolicyFactory(int maxSize, int batchSize, long maxMemSize) {
+        setMaxSize(maxSize);
+        setBatchSize(batchSize);
+        setMaxMemorySize(maxMemSize);
+    }
+
+    /** {@inheritDoc} */
+    @Override public FifoEvictionPolicy<K, V> create() {
+        FifoEvictionPolicy<K, V> policy = new FifoEvictionPolicy<>();
+
+        policy.setBatchSize(getBatchSize());
+        policy.setMaxMemorySize(getMaxMemorySize());
+        policy.setMaxSize(getMaxSize());
+
+        return policy;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/lru/LruEvictionPolicyFactory.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/lru/LruEvictionPolicyFactory.java
new file mode 100644
index 0000000000000..8f7fbc5c7b95d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/lru/LruEvictionPolicyFactory.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.eviction.lru;
+
+import org.apache.ignite.cache.eviction.AbstractEvictionPolicyFactory;
+
+/**
+ * Factory class for {@link LruEvictionPolicy}.
+ *
+ * Creates cache Eviction policy based on {@code Least Recently Used (LRU)} algorithm and supports batch eviction.
+ * <p>
+ * The eviction starts in the following cases:
+ * <ul>
+ *     <li>The cache size becomes {@code batchSize} elements greater than the maximum size.</li>
+ *     <li>
+ *         The size of cache entries in bytes becomes greater than the maximum memory size.
+ *         The size of cache entry calculates as sum of key size and value size.
+ *     </li>
+ * </ul>
+ * <b>Note:</b>Batch eviction is enabled only if maximum memory limit isn't set ({@code maxMemSize == 0}).
+ * {@code batchSize} elements will be evicted in this case. The default {@code batchSize} value is {@code 1}.
+
+ *  {@link LruEvictionPolicy} implementation is very efficient since it is lock-free and does not create any additional table-like
+ * data structures. The {@code LRU} ordering information is maintained by attaching ordering metadata to cache entries.
+ */
+public class LruEvictionPolicyFactory<K, V> extends AbstractEvictionPolicyFactory<LruEvictionPolicy<K, V>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    public LruEvictionPolicyFactory() {
+    }
+
+    /** */
+    public LruEvictionPolicyFactory(int maxSize) {
+        setMaxSize(maxSize);
+    }
+
+    /** */
+    public LruEvictionPolicyFactory(int maxSize, int batchSize, long maxMemSize) {
+        setMaxSize(maxSize);
+        setBatchSize(batchSize);
+        setMaxMemorySize(maxMemSize);
+    }
+
+    /** {@inheritDoc} */
+    @Override public LruEvictionPolicy<K, V> create() {
+        LruEvictionPolicy<K, V> policy = new LruEvictionPolicy<>();
+
+        policy.setBatchSize(getBatchSize());
+        policy.setMaxMemorySize(getMaxMemorySize());
+        policy.setMaxSize(getMaxSize());
+
+        return policy;
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/sorted/SortedEvictionPolicyFactory.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/sorted/SortedEvictionPolicyFactory.java
new file mode 100644
index 0000000000000..a88c277485eb7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/sorted/SortedEvictionPolicyFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.eviction.sorted;
+
+import java.io.Serializable;
+import java.util.Comparator;
+import org.apache.ignite.cache.eviction.AbstractEvictionPolicyFactory;
+import org.apache.ignite.cache.eviction.EvictableEntry;
+
+/**
+ * Factory class for {@link SortedEvictionPolicy}.
+ *
+ * Creates cache Eviction policy which will select the minimum cache entry for eviction.
+ * <p>
+ * The eviction starts in the following cases:
+ * <ul>
+ *     <li>The cache size becomes {@code batchSize} elements greater than the maximum size.</li>
+ *     <li>
+ *         The size of cache entries in bytes becomes greater than the maximum memory size.
+ *         The size of cache entry calculates as sum of key size and value size.
+ *     </li>
+ * </ul>
+ * <b>Note:</b>Batch eviction is enabled only if maximum memory limit isn't set ({@code maxMemSize == 0}).
+ * {@code batchSize} elements will be evicted in this case. The default {@code batchSize} value is {@code 1}.
+ * <p>
+ * Entries comparison based on {@link Comparator} instance if provided.
+ * Default {@code Comparator} behaviour is use cache entries keys for comparison that imposes a requirement for keys
+ * to implement {@link Comparable} interface.
+ * <p>
+ * User defined comparator should implement {@link Serializable} interface.
+ */
+public class SortedEvictionPolicyFactory<K,V> extends AbstractEvictionPolicyFactory<SortedEvictionPolicy<K, V>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Comparator. */
+    private Comparator<EvictableEntry<K, V>> comp;
+
+    /** */
+    public SortedEvictionPolicyFactory() {
+    }
+
+    /** */
+    public SortedEvictionPolicyFactory(int maxSize) {
+        setMaxSize(maxSize);
+    }
+
+    /** */
+    public SortedEvictionPolicyFactory(int maxSize, int batchSize, long maxMemSize) {
+        setMaxSize(maxSize);
+        setBatchSize(batchSize);
+        setMaxMemorySize(maxMemSize);
+    }
+
+    /**
+     * Gets entries comparator.
+     * @return entry comparator.
+     */
+    public Comparator<EvictableEntry<K, V>> getComp() {
+        return comp;
+    }
+
+    /**
+     * Sets entries comparator.
+     *
+     * @param comp entry comparator.
+     */
+    public void setComp(Comparator<EvictableEntry<K, V>> comp) {
+        this.comp = comp;
+    }
+
+    /** {@inheritDoc} */
+    @Override public SortedEvictionPolicy<K, V> create() {
+        SortedEvictionPolicy<K, V> policy = new SortedEvictionPolicy<>(comp);
+
+        policy.setBatchSize(getBatchSize());
+        policy.setMaxMemorySize(getMaxMemorySize());
+        policy.setMaxSize(getMaxSize());
+
+        return policy;
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
index 31806e73e69e7..cf6cd4e1ba28b 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
@@ -242,9 +242,13 @@ public class CacheConfiguration<K, V> extends MutableConfiguration<K, V> {
     /** Rebalance timeout. */
     private long rebalanceTimeout = DFLT_REBALANCE_TIMEOUT;
 
-    /** Cache expiration policy. */
+    /** Cache eviction policy. */
+    @Deprecated
     private EvictionPolicy evictPlc;
 
+    /** Cache eviction policy factory. */
+    private Factory evictPlcFactory;
+
     /** Flag indicating whether eviction is synchronized. */
     private boolean evictSync = DFLT_EVICT_SYNCHRONIZED;
 
@@ -466,6 +470,7 @@ public CacheConfiguration(CompleteConfiguration<K, V> cfg) {
         evictKeyBufSize = cc.getEvictSynchronizedKeyBufferSize();
         evictMaxOverflowRatio = cc.getEvictMaxOverflowRatio();
         evictPlc = cc.getEvictionPolicy();
+        evictPlcFactory = cc.getEvictionPolicyFactory();
         evictSync = cc.isEvictSynchronized();
         evictSyncConcurrencyLvl = cc.getEvictSynchronizedConcurrencyLevel();
         evictSyncTimeout = cc.getEvictSynchronizedTimeout();
@@ -549,7 +554,10 @@ public CacheConfiguration<K, V> setName(String name) {
      * which means that evictions are disabled for cache.
      *
      * @return Cache eviction policy or {@code null} if evictions should be disabled.
+     *
+     * @deprecated Use {@link #getEvictionPolicyFactory()} instead.
      */
+    @Deprecated
     @SuppressWarnings({"unchecked"})
     @Nullable public EvictionPolicy<K, V> getEvictionPolicy() {
         return evictPlc;
@@ -558,15 +566,43 @@ public CacheConfiguration<K, V> setName(String name) {
     /**
      * Sets cache eviction policy.
      *
-     * @param evictPlc Cache expiration policy.
+     * @param evictPlc Cache eviction policy.
      * @return {@code this} for chaining.
+     *
+     * @deprecated Use {@link #setEvictionPolicyFactory(Factory)} instead.
      */
+    @Deprecated
     public CacheConfiguration<K, V> setEvictionPolicy(@Nullable EvictionPolicy evictPlc) {
         this.evictPlc = evictPlc;
 
         return this;
     }
 
+    /**
+     * Gets cache eviction policy factory. By default, returns {@code null}
+     * which means that evictions are disabled for cache.
+     *
+     * @return Cache eviction policy factory or {@code null} if evictions should be disabled
+     * or if {@link #getEvictionPolicy()} should be used instead.
+     */
+    @Nullable public Factory<EvictionPolicy<? super K, ? super V>> getEvictionPolicyFactory() {
+        return evictPlcFactory;
+    }
+
+    /**
+     * Sets cache eviction policy factory.
+     * Note: Eviction policy factory should be {@link Serializable}.
+     *
+     * @param evictPlcFactory Cache eviction policy factory.
+     * @return {@code this} for chaining.
+     */
+    public CacheConfiguration<K, V> setEvictionPolicyFactory(
+        @Nullable Factory<? extends EvictionPolicy<? super K, ? super V>> evictPlcFactory) {
+        this.evictPlcFactory = evictPlcFactory;
+
+        return this;
+    }
+
     /**
      * @return Near enabled flag.
      */
@@ -786,7 +822,7 @@ public CacheConfiguration<K, V> setEvictMaxOverflowRatio(float evictMaxOverflowR
      * never be evicted.
      * <p>
      * If not provided, any entry may be evicted depending on
-     * {@link #getEvictionPolicy() eviction policy} configuration.
+     * {@link #getEvictionPolicyFactory()} eviction policy} configuration.
      *
      * @return Eviction filter or {@code null}.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/NearCacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/NearCacheConfiguration.java
index 3c59bd6d7db07..e9c71f941ff73 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/NearCacheConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/NearCacheConfiguration.java
@@ -17,10 +17,13 @@
 
 package org.apache.ignite.configuration;
 
+import java.io.Serializable;
 import javax.cache.configuration.MutableConfiguration;
+import javax.cache.configuration.Factory;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.eviction.EvictionPolicy;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.configuration.CacheConfiguration.DFLT_NEAR_START_SIZE;
 
@@ -37,8 +40,12 @@ public class NearCacheConfiguration<K, V> extends MutableConfiguration<K, V> {
     private static final long serialVersionUID = 0L;
 
     /** Near cache eviction policy. */
+    @Deprecated
     private EvictionPolicy<K, V> nearEvictPlc;
 
+    /** Near cache eviction policy factory. */
+    private Factory nearEvictPlcFactory;
+
     /** Default near cache start size. */
     private int nearStartSize = DFLT_NEAR_START_SIZE;
 
@@ -57,6 +64,7 @@ public NearCacheConfiguration() {
     public NearCacheConfiguration(NearCacheConfiguration<K, V> ccfg) {
         super(ccfg);
 
+        nearEvictPlcFactory = ccfg.getNearEvictionPolicyFactory();
         nearEvictPlc = ccfg.getNearEvictionPolicy();
         nearStartSize = ccfg.getNearStartSize();
     }
@@ -68,7 +76,10 @@ public NearCacheConfiguration(NearCacheConfiguration<K, V> ccfg) {
      * @return Near eviction policy.
      * @see CacheConfiguration#getEvictionPolicy()
      * @see CacheConfiguration#isEvictSynchronized()
+     *
+     * @deprecated Use {@link #getNearEvictionPolicyFactory()}  instead.
      */
+    @Deprecated
     public EvictionPolicy<K, V> getNearEvictionPolicy() {
         return nearEvictPlc;
     }
@@ -78,13 +89,43 @@ public EvictionPolicy<K, V> getNearEvictionPolicy() {
      *
      * @param nearEvictPlc Near eviction policy.
      * @return {@code this} for chaining.
+     *
+     * @deprecated Use {@link #setNearEvictionPolicyFactory(Factory)} instead.
      */
+    @Deprecated
     public NearCacheConfiguration<K, V> setNearEvictionPolicy(EvictionPolicy<K, V> nearEvictPlc) {
         this.nearEvictPlc = nearEvictPlc;
 
         return this;
     }
 
+    /**
+     * Gets cache eviction policy factory. By default, returns {@code null}
+     * which means that evictions are disabled for cache.
+     *
+     * @return Cache eviction policy or {@code null} if evictions should be disabled.
+     */
+    @Nullable public Factory<EvictionPolicy<? super K, ? super V>> getNearEvictionPolicyFactory() {
+        return nearEvictPlcFactory;
+    }
+
+    /**
+     * Sets cache eviction policy factory.
+     * Note: Eviction policy factory should be {@link Serializable}.
+     *
+     * @see CacheConfiguration#getEvictionPolicyFactory()
+     * @see CacheConfiguration#isEvictSynchronized()
+     *
+     * @param nearEvictPlcFactory Cache eviction policy.
+     * @return {@code this} for chaining.
+     */
+    public NearCacheConfiguration<K, V> setNearEvictionPolicyFactory(
+        @Nullable Factory<? extends EvictionPolicy<? super K, ? super V>> nearEvictPlcFactory) {
+        this.nearEvictPlcFactory = nearEvictPlcFactory;
+
+        return this;
+    }
+
     /**
      * Gets initial cache size for near cache which will be used to pre-create internal
      * hash table after start. Default value is defined by {@link CacheConfiguration#DFLT_NEAR_START_SIZE}.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java
index 57f12f635489b..568886b0dfde4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java
@@ -170,13 +170,25 @@ public String evictionFilterClassName() {
 
     /**
      * @return Eviction policy class name.
+     *
+     * @deprecated Use evictionPolicyFactoryClassName() instead.
      */
+    @Deprecated
     public String evictionPolicyClassName() {
         return className(ccfg.getEvictionPolicy());
     }
 
+    /**
+     * @return Eviction policy factory class name.
+     */
+    public String evictionPolicyFactoryClassName() {
+        return className(ccfg.getEvictionPolicyFactory());
+    }
+
     /**
      * @return Near eviction policy class name.
+     *
+     * @deprecated Use nearEvictionPolicyFactoryClassName() instead.
      */
     public String nearEvictionPolicyClassName() {
         NearCacheConfiguration nearCfg = ccfg.getNearConfiguration();
@@ -187,6 +199,13 @@ public String nearEvictionPolicyClassName() {
         return className(nearCfg.getNearEvictionPolicy());
     }
 
+    /**
+     * @return Near eviction policy factory class name.
+     */
+    public String nearEvictionPolicyFactoryClassName() {
+        return className(ccfg.getEvictionPolicyFactory());
+    }
+
     /**
      * @return Store class name.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
index f8722d6a4f338..ed2bd6ff3f548 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
@@ -170,7 +170,15 @@ public class GridCacheEvictionManager extends GridCacheManagerAdapter {
     @Override public void start0() throws IgniteCheckedException {
         CacheConfiguration cfg = cctx.config();
 
-        plc = cctx.isNear() ? cfg.getNearConfiguration().getNearEvictionPolicy() : cfg.getEvictionPolicy();
+        if (cctx.isNear()) {
+            plc = (cfg.getNearConfiguration().getNearEvictionPolicyFactory() != null) ?
+                (EvictionPolicy)cfg.getNearConfiguration().getNearEvictionPolicyFactory().create() :
+                cfg.getNearConfiguration().getNearEvictionPolicy();
+        }
+        else if (cfg.getEvictionPolicyFactory() != null)
+            plc = (EvictionPolicy)cfg.getEvictionPolicyFactory().create();
+        else
+            plc = cfg.getEvictionPolicy();
 
         memoryMode = cctx.config().getMemoryMode();
 
@@ -2119,4 +2127,9 @@ Collection<IgniteBiTuple<ClusterNode, Long>> evictedReaders(KeyCacheObject key)
             return S.toString(EvictionFuture.class, this);
         }
     }
+
+    /** For test purposes. */
+    public EvictionPolicy getEvictionPolicy() {
+        return plc;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 8a041b2a1a4ab..e9065603ed5ff 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -356,7 +356,7 @@ private void suggestOptimizations(CacheConfiguration cfg, boolean hasStore) {
 
         String msg = "Disable eviction policy (remove from configuration)";
 
-        if (cfg.getEvictionPolicy() != null) {
+        if (cfg.getEvictionPolicyFactory() != null || cfg.getEvictionPolicy() != null) {
             perf.add(msg, false);
 
             perf.add("Disable synchronized evictions (set 'evictSynchronized' to false)", !cfg.isEvictSynchronized());
@@ -482,7 +482,8 @@ else if (cc.getRebalanceMode() == SYNC) {
 
         switch (cc.getMemoryMode()) {
             case OFFHEAP_VALUES: {
-                if (cacheType.userCache() && cc.getEvictionPolicy() == null && cc.getOffHeapMaxMemory() >= 0)
+                if (cacheType.userCache() && cc.getEvictionPolicyFactory() == null
+                    && cc.getEvictionPolicy() == null && cc.getOffHeapMaxMemory() >= 0)
                     U.quietAndWarn(log, "Off heap maximum memory configuration property will be ignored for the " +
                         "cache working in OFFHEAP_VALUES mode (memory usage will be unlimited): " +
                         U.maskName(cc.getName()) + ". Consider configuring eviction policy or switching to " +
@@ -502,7 +503,8 @@ else if (cc.getRebalanceMode() == SYNC) {
             }
 
             case ONHEAP_TIERED:
-                if (cacheType.userCache() && cc.getEvictionPolicy() == null && cc.getOffHeapMaxMemory() >= 0)
+                if (cacheType.userCache() && cc.getEvictionPolicyFactory() == null
+                    && cc.getEvictionPolicy() == null && cc.getOffHeapMaxMemory() >= 0)
                     U.quietAndWarn(log, "Eviction policy not enabled with ONHEAP_TIERED mode for cache " +
                         "(entries will not be moved to off-heap store): " + U.maskName(cc.getName()));
 
@@ -551,6 +553,7 @@ private Collection<GridCacheManager> dhtExcludes(GridCacheContext ctx) {
      * @throws IgniteCheckedException If failed to inject.
      */
     private void prepare(CacheConfiguration cfg, Collection<Object> objs) throws IgniteCheckedException {
+        prepare(cfg, cfg.getEvictionPolicyFactory(), false);
         prepare(cfg, cfg.getEvictionPolicy(), false);
         prepare(cfg, cfg.getAffinity(), false);
         prepare(cfg, cfg.getAffinityMapper(), false);
@@ -560,8 +563,10 @@ private void prepare(CacheConfiguration cfg, Collection<Object> objs) throws Ign
 
         NearCacheConfiguration nearCfg = cfg.getNearConfiguration();
 
-        if (nearCfg != null)
+        if (nearCfg != null) {
+            prepare(cfg, nearCfg.getNearEvictionPolicyFactory(), true);
             prepare(cfg, nearCfg.getNearEvictionPolicy(), true);
+        }
 
         for (Object obj : objs)
             prepare(cfg, obj, false);
@@ -589,6 +594,7 @@ private void prepare(CacheConfiguration cfg, @Nullable Object rsrc, boolean near
     private void cleanup(GridCacheContext cctx) {
         CacheConfiguration cfg = cctx.config();
 
+        cleanup(cfg, cfg.getEvictionPolicyFactory(), false);
         cleanup(cfg, cfg.getEvictionPolicy(), false);
         cleanup(cfg, cfg.getAffinity(), false);
         cleanup(cfg, cfg.getAffinityMapper(), false);
@@ -604,8 +610,10 @@ private void cleanup(GridCacheContext cctx) {
 
         NearCacheConfiguration nearCfg = cfg.getNearConfiguration();
 
-        if (nearCfg != null)
+        if (nearCfg != null) {
+            cleanup(cfg, nearCfg.getNearEvictionPolicyFactory(), true);
             cleanup(cfg, nearCfg.getNearEvictionPolicy(), true);
+        }
 
         cctx.cleanup();
     }
@@ -2983,6 +2991,10 @@ private void checkCache(CacheConfiguration locCfg, CacheConfiguration rmtCfg, Cl
             CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "evictionPolicy", "Eviction policy",
                 locAttr.evictionPolicyClassName(), rmtAttr.evictionPolicyClassName(), true);
 
+            CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "evictionPolicyFactory",
+                "Eviction policy factory", locAttr.evictionPolicyFactoryClassName(),
+                rmtAttr.evictionPolicyFactoryClassName(), true);
+
             CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "transactionManagerLookup",
                 "Transaction manager lookup", locAttr.transactionManagerLookupClassName(),
                 rmtAttr.transactionManagerLookupClassName(), false);
@@ -3036,6 +3048,10 @@ private void checkCache(CacheConfiguration locCfg, CacheConfiguration rmtCfg, Cl
                     "Near eviction policy", locAttr.nearEvictionPolicyClassName(),
                     rmtAttr.nearEvictionPolicyClassName(), false);
 
+                CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "nearEvictionPolicyFactory",
+                    "Near eviction policy factory", locAttr.nearEvictionPolicyFactoryClassName(),
+                    rmtAttr.nearEvictionPolicyFactoryClassName(), false);
+
                 CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "affinityIncludeNeighbors",
                     "Affinity include neighbors", locAttr.affinityIncludeNeighbors(),
                     rmtAttr.affinityIncludeNeighbors(), true);
@@ -3671,14 +3687,17 @@ private Iterable<Object> lifecycleAwares(CacheConfiguration ccfg, Object... objs
         ret.add(ccfg.getAffinity());
         ret.add(ccfg.getAffinityMapper());
         ret.add(ccfg.getEvictionFilter());
+        ret.add(ccfg.getEvictionPolicyFactory());
         ret.add(ccfg.getEvictionPolicy());
         ret.add(ccfg.getInterceptor());
         ret.add(ccfg.getTopologyValidator());
 
         NearCacheConfiguration nearCfg = ccfg.getNearConfiguration();
 
-        if (nearCfg != null)
+        if (nearCfg != null) {
+            ret.add(nearCfg.getNearEvictionPolicyFactory());
             ret.add(nearCfg.getNearEvictionPolicy());
+        }
 
         Collections.addAll(ret, objs);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index e274485ec45f6..2a976ecbe230c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1145,6 +1145,7 @@ public static CacheConfiguration hadoopSystemCache() {
         cache.setAtomicityMode(TRANSACTIONAL);
         cache.setWriteSynchronizationMode(FULL_SYNC);
 
+        cache.setEvictionPolicyFactory(null);
         cache.setEvictionPolicy(null);
         cache.setSwapEnabled(false);
         cache.setCacheStoreFactory(null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHelperImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHelperImpl.java
index 29e75a5a122d2..f20b787aa7c17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHelperImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHelperImpl.java
@@ -31,7 +31,9 @@
 public class IgfsHelperImpl implements IgfsHelper {
     /** {@inheritDoc} */
     @Override public void preProcessCacheConfiguration(CacheConfiguration cfg) {
-        EvictionPolicy evictPlc = cfg.getEvictionPolicy();
+        EvictionPolicy evictPlc = cfg.getEvictionPolicyFactory() != null ?
+            (EvictionPolicy)cfg.getEvictionPolicyFactory().create()
+            : cfg.getEvictionPolicy();
 
         if (evictPlc instanceof IgfsPerBlockLruEvictionPolicy && cfg.getEvictionFilter() == null)
             cfg.setEvictionFilter(new IgfsEvictionFilter());
@@ -39,7 +41,9 @@ public class IgfsHelperImpl implements IgfsHelper {
 
     /** {@inheritDoc} */
     @Override public void validateCacheConfiguration(CacheConfiguration cfg) throws IgniteCheckedException {
-        EvictionPolicy evictPlc =  cfg.getEvictionPolicy();
+        EvictionPolicy evictPlc = cfg.getEvictionPolicyFactory() != null ?
+            (EvictionPolicy)cfg.getEvictionPolicyFactory().create()
+            : cfg.getEvictionPolicy();
 
         if (evictPlc != null && evictPlc instanceof IgfsPerBlockLruEvictionPolicy) {
             EvictionFilter evictFilter = cfg.getEvictionFilter();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 6b23e8084a46b..98313bad4dbbe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -263,7 +263,9 @@ public final class IgfsImpl implements IgfsEx {
 
         for (CacheConfiguration cacheCfg : igfsCtx.kernalContext().config().getCacheConfiguration()) {
             if (F.eq(dataCacheName, cacheCfg.getName())) {
-                EvictionPolicy evictPlc = cacheCfg.getEvictionPolicy();
+                EvictionPolicy evictPlc = cacheCfg.getEvictionPolicyFactory() != null ?
+                    (EvictionPolicy)cacheCfg.getEvictionPolicyFactory().create()
+                    : cacheCfg.getEvictionPolicy();
 
                 if (evictPlc != null & evictPlc instanceof IgfsPerBlockLruEvictionPolicy)
                     this.evictPlc = (IgfsPerBlockLruEvictionPolicy)evictPlc;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheConfigurationConsistencySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheConfigurationConsistencySelfTest.java
index a1f917fccb8a4..07072f7b1c5e3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheConfigurationConsistencySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheConfigurationConsistencySelfTest.java
@@ -30,8 +30,11 @@
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cache.eviction.EvictionFilter;
 import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy;
+import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicyFactory;
 import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy;
+import org.apache.ignite.cache.eviction.lru.LruEvictionPolicyFactory;
 import org.apache.ignite.cache.eviction.random.RandomEvictionPolicy;
+import org.apache.ignite.cache.eviction.sorted.SortedEvictionPolicyFactory;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DeploymentMode;
@@ -374,6 +377,27 @@ public void testDifferentEvictionEnabled() throws Exception {
         );
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDifferentEvictionPolicyEnabled() throws Exception {
+        checkSecondGridStartFails(
+            new C1<CacheConfiguration, Void>() {
+                /** {@inheritDoc} */
+                @Override public Void apply(CacheConfiguration cfg) {
+                    cfg.setEvictionPolicyFactory(new FifoEvictionPolicyFactory<>());
+                    return null;
+                }
+            },
+            new C1<CacheConfiguration, Void>() {
+                /** {@inheritDoc} */
+                @Override public Void apply(CacheConfiguration cfg) {
+                    return null;
+                }
+            }
+        );
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -396,6 +420,28 @@ public void testDifferentEvictionPolicies() throws Exception {
         );
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDifferentEvictionPolicyFactories() throws Exception {
+        checkSecondGridStartFails(
+            new C1<CacheConfiguration, Void>() {
+                /** {@inheritDoc} */
+                @Override public Void apply(CacheConfiguration cfg) {
+                    cfg.setEvictionPolicyFactory(new SortedEvictionPolicyFactory());
+                    return null;
+                }
+            },
+            new C1<CacheConfiguration, Void>() {
+                /** {@inheritDoc} */
+                @Override public Void apply(CacheConfiguration cfg) {
+                    cfg.setEvictionPolicyFactory(new FifoEvictionPolicyFactory<>());
+                    return null;
+                }
+            }
+        );
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -632,6 +678,7 @@ public void testPartitionedOnlyAttributesIgnoredForReplicated() throws Exception
             @Override public Void apply(CacheConfiguration cfg) {
                 NearCacheConfiguration nearCfg = new NearCacheConfiguration();
 
+                nearCfg.setNearEvictionPolicyFactory(new LruEvictionPolicyFactory<>());
                 nearCfg.setNearEvictionPolicy(new RandomEvictionPolicy());
 
                 cfg.setNearConfiguration(nearCfg);
@@ -646,6 +693,7 @@ public void testPartitionedOnlyAttributesIgnoredForReplicated() throws Exception
             @Override public Void apply(CacheConfiguration cfg) {
                 NearCacheConfiguration nearCfg = new NearCacheConfiguration();
 
+                nearCfg.setNearEvictionPolicyFactory(new FifoEvictionPolicyFactory<>());
                 nearCfg.setNearEvictionPolicy(new FifoEvictionPolicy());
 
                 cfg.setNearConfiguration(nearCfg);
@@ -671,6 +719,7 @@ public void testIgnoreMismatchForLocalCaches() throws Exception {
             @Override public Void apply(CacheConfiguration cfg) {
                 cfg.setAffinity(new TestRendezvousAffinityFunction());
 
+                cfg.setEvictionPolicyFactory(new FifoEvictionPolicyFactory<>());
                 cfg.setEvictionPolicy(new FifoEvictionPolicy());
 
                 cfg.setCacheStoreFactory(new IgniteCacheAbstractTest.TestStoreFactory());
@@ -689,6 +738,7 @@ public void testIgnoreMismatchForLocalCaches() throws Exception {
             @Override public Void apply(CacheConfiguration cfg) {
                 cfg.setAffinity(new RendezvousAffinityFunction());
 
+                cfg.setEvictionPolicyFactory(new FifoEvictionPolicyFactory<>());
                 cfg.setEvictionPolicy(new LruEvictionPolicy());
 
                 cfg.setCacheStoreFactory(null);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionEventSelfTest.java
index 7088ad7c43fc8..0d36a5a7fde6b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionEventSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearEvictionEventSelfTest.java
@@ -36,9 +36,4 @@ public class GridCacheNearEvictionEventSelfTest extends GridCacheEvictionEventAb
     @Override protected CacheAtomicityMode atomicityMode() {
         return TRANSACTIONAL;
     }
-
-    /** {@inheritDoc} */
-    @Override public void testEvictionEvent() throws Exception {
-        super.testEvictionEvent();
-    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/EvictionPolicyFactoryAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/EvictionPolicyFactoryAbstractTest.java
new file mode 100644
index 0000000000000..89d7f0b42ef6d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/EvictionPolicyFactoryAbstractTest.java
@@ -0,0 +1,1074 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.eviction;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.eviction.EvictableEntry;
+import org.apache.ignite.cache.eviction.EvictionFilter;
+import org.apache.ignite.cache.eviction.EvictionPolicy;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheEvictionManager;
+import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
+import org.apache.ignite.internal.util.typedef.C2;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.LOCAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.events.EventType.EVT_JOB_MAPPED;
+import static org.apache.ignite.events.EventType.EVT_TASK_FAILED;
+import static org.apache.ignite.events.EventType.EVT_TASK_FINISHED;
+import static org.apache.ignite.internal.processors.cache.eviction.EvictionPolicyFactoryAbstractTest.EvictionPolicyProxy.proxy;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Base class for eviction tests.
+ */
+public abstract class EvictionPolicyFactoryAbstractTest<T extends EvictionPolicy<?, ?>>
+    extends GridCommonAbstractTest {
+    /** IP finder. */
+    protected static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final String DEFAULT_CACHE_NAME = null;
+
+    /** Put entry size. */
+    protected static final int PUT_ENTRY_SIZE = 10;
+
+    /** Replicated cache. */
+    protected CacheMode mode = REPLICATED;
+
+    /** Near enabled flag. */
+    protected boolean nearEnabled;
+
+    /** Policy max. */
+    protected int plcMax = 10;
+
+    /** Policy batch size. */
+    protected int plcBatchSize = 1;
+
+    /** Policy max memory size. */
+    protected long plcMaxMemSize = 0;
+
+    protected Factory<T> policyFactory;
+
+    /** Near policy max. */
+    protected int nearMax = 3;
+
+    /** Synchronous commit. */
+    protected boolean syncCommit;
+
+    /** */
+    protected int gridCnt = 2;
+
+    /** */
+    protected EvictionFilter<?, ?> filter;
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        policyFactory = null;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        assert policyFactory != null;
+
+        IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
+
+        CacheConfiguration cc = defaultCacheConfiguration();
+
+        cc.setCacheMode(mode);
+        cc.setEvictionPolicyFactory(policyFactory);
+        cc.setWriteSynchronizationMode(syncCommit ? FULL_SYNC : FULL_ASYNC);
+        cc.setAtomicityMode(TRANSACTIONAL);
+
+        if (nearEnabled) {
+            NearCacheConfiguration nearCfg = new NearCacheConfiguration();
+
+            nearCfg.setNearEvictionPolicyFactory(createNearPolicyFactory(nearMax));
+
+            cc.setNearConfiguration(nearCfg);
+        }
+        else
+            cc.setNearConfiguration(null);
+
+        if (mode == PARTITIONED)
+            cc.setBackups(1);
+
+        if (filter != null)
+            cc.setEvictionFilter(filter);
+
+        c.setCacheConfiguration(cc);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(ipFinder);
+
+        c.setDiscoverySpi(disco);
+
+        c.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
+
+        c.setIncludeProperties();
+
+        return c;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        filter = null;
+
+        super.afterTestsStopped();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxSizePolicy() throws Exception {
+        plcMax = 3;
+        plcMaxMemSize = 0;
+        plcBatchSize = 1;
+
+        doTestPolicy();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxSizePolicyWithBatch() throws Exception {
+        plcMax = 3;
+        plcMaxMemSize = 0;
+        plcBatchSize = 2;
+
+        doTestPolicyWithBatch();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxMemSizePolicy() throws Exception {
+        plcMax = 0;
+        plcMaxMemSize = 3 * MockEntry.ENTRY_SIZE;
+        plcBatchSize = 1;
+
+        doTestPolicy();
+    }
+
+    /**
+     * Batch ignored when {@code maxSize > 0} and {@code maxMemSize > 0}.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMaxMemSizePolicyWithBatch() throws Exception {
+        plcMax = 3;
+        plcMaxMemSize = 10 * MockEntry.ENTRY_SIZE;
+        plcBatchSize = 2;
+
+        doTestPolicy();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxSizeMemory() throws Exception {
+        int max = 10;
+
+        plcMax = max;
+        plcMaxMemSize = 0;
+        plcBatchSize = 1;
+
+        doTestMemory(max);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxSizeMemoryWithBatch() throws Exception {
+        int max = 10;
+
+        plcMax = max;
+        plcMaxMemSize = 0;
+        plcBatchSize = 2;
+
+        doTestMemory(max);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxMemSizeMemory() throws Exception {
+        int max = 10;
+
+        plcMax = 0;
+        plcMaxMemSize = max * MockEntry.ENTRY_SIZE;
+        plcBatchSize = 1;
+
+        doTestMemory(max);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxSizeRandom() throws Exception {
+        plcMax = 10;
+        plcMaxMemSize = 0;
+        plcBatchSize = 1;
+
+        doTestRandom();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxSizeRandomWithBatch() throws Exception {
+        plcMax = 10;
+        plcMaxMemSize = 0;
+        plcBatchSize = 2;
+
+        doTestRandom();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxMemSizeRandom() throws Exception {
+        plcMax = 0;
+        plcMaxMemSize = 10 * MockEntry.KEY_SIZE;
+        plcBatchSize = 1;
+
+        doTestRandom();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxSizeAllowEmptyEntries() throws Exception {
+        plcMax = 10;
+        plcMaxMemSize = 0;
+        plcBatchSize = 1;
+
+        doTestAllowEmptyEntries();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxSizeAllowEmptyEntriesWithBatch() throws Exception {
+        plcMax = 10;
+        plcMaxMemSize = 0;
+        plcBatchSize = 2;
+
+        doTestAllowEmptyEntries();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxMemSizeAllowEmptyEntries() throws Exception {
+        plcMax = 0;
+        plcMaxMemSize = 10 * MockEntry.KEY_SIZE;
+        plcBatchSize = 1;
+
+        doTestAllowEmptyEntries();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxSizePut() throws Exception {
+        plcMax = 100;
+        plcBatchSize = 1;
+        plcMaxMemSize = 0;
+
+        doTestPut(plcMax);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxSizePutWithBatch() throws Exception {
+        plcMax = 100;
+        plcBatchSize = 2;
+        plcMaxMemSize = 0;
+
+        doTestPut(plcMax);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMaxMemSizePut() throws Exception {
+        int max = 100;
+
+        plcMax = 0;
+        plcBatchSize = 2;
+        plcMaxMemSize = max * PUT_ENTRY_SIZE;
+
+        doTestPut(max);
+    }
+
+    /**
+     * Tests policy behaviour.
+     *
+     * @throws Exception If failed.
+     */
+    protected abstract void doTestPolicy() throws Exception;
+
+    /**
+     * Tests policy behaviour with batch enabled.
+     *
+     * @throws Exception If failed.
+     */
+    protected abstract void doTestPolicyWithBatch() throws Exception;
+
+    /**
+     * @throws Exception If failed.
+     */
+    protected void doTestAllowEmptyEntries() throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGrid();
+
+            MockEntry e1 = new MockEntry("1");
+            MockEntry e2 = new MockEntry("2");
+            MockEntry e3 = new MockEntry("3");
+            MockEntry e4 = new MockEntry("4");
+            MockEntry e5 = new MockEntry("5");
+
+            EvictionPolicyProxy p = proxy(policy());
+
+            p.onEntryAccessed(false, e1);
+
+            assertFalse(e1.isEvicted());
+
+            check(p.queue().size(), MockEntry.KEY_SIZE);
+
+            p.onEntryAccessed(false, e2);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e2.isEvicted());
+
+            check(p.queue().size(), MockEntry.KEY_SIZE);
+
+            p.onEntryAccessed(false, e3);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e3.isEvicted());
+
+            check(p.queue().size(), MockEntry.KEY_SIZE);
+
+            p.onEntryAccessed(false, e4);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+
+            check(p.queue().size(), MockEntry.KEY_SIZE);
+
+            p.onEntryAccessed(false, e5);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            check(p.queue().size(), MockEntry.KEY_SIZE);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    protected void doTestMemory(int max) throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGrid();
+
+            EvictionPolicyProxy p = proxy(policy());
+
+            int cnt = max + plcBatchSize;
+
+            for (int i = 0; i < cnt; i++)
+                p.onEntryAccessed(false, new MockEntry(Integer.toString(i), Integer.toString(i)));
+
+            info(p);
+
+            check(max, MockEntry.ENTRY_SIZE);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    protected void doTestRandom() throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGrid();
+
+            EvictionPolicyProxy p = proxy(policy());
+
+            int max = 10;
+
+            Random rand = new Random();
+
+            int keys = 31;
+
+            MockEntry[] entries = new MockEntry[keys];
+
+            for (int i = 0; i < entries.length; i++)
+                entries[i] = new MockEntry(Integer.toString(i));
+
+            int runs = 5000000;
+
+            for (int i = 0; i < runs; i++) {
+                boolean rmv = rand.nextBoolean();
+
+                int j = rand.nextInt(entries.length);
+
+                MockEntry e = entry(entries, j);
+
+                if (rmv)
+                    entries[j] = new MockEntry(Integer.toString(j));
+
+                p.onEntryAccessed(rmv, e);
+            }
+
+            info(p);
+
+            assertTrue(p.getCurrentSize() <= (plcMaxMemSize > 0 ? max : max + plcBatchSize));
+            assertTrue(p.getCurrentMemorySize() <= (plcMaxMemSize > 0 ? max : max + plcBatchSize) * MockEntry.KEY_SIZE);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    protected void doTestPut(int max) throws Exception {
+        mode = LOCAL;
+        syncCommit = true;
+
+        policyFactory = createPolicyFactory();
+
+        try {
+            Ignite ignite = startGrid();
+
+            IgniteCache<Object, Object> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+            int cnt = 500;
+
+            int min = Integer.MAX_VALUE;
+
+            int minIdx = 0;
+
+            for (int i = 0; i < cnt; i++) {
+                cache.put(i, i);
+
+                int cacheSize = cache.size();
+
+                if (i > max && cacheSize < min) {
+                    min = cacheSize;
+                    minIdx = i;
+                }
+            }
+
+            assertTrue("Min cache size is too small: " + min, min >= max);
+
+            check(max, PUT_ENTRY_SIZE);
+
+            info("Min cache size [min=" + min + ", idx=" + minIdx + ']');
+            info("Current cache size " + cache.size());
+            info("Current cache key size " + cache.size());
+
+            min = Integer.MAX_VALUE;
+
+            minIdx = 0;
+
+            // Touch.
+            for (int i = cnt; --i > cnt - max;) {
+                cache.get(i);
+
+                int cacheSize = cache.size();
+
+                if (cacheSize < min) {
+                    min = cacheSize;
+                    minIdx = i;
+                }
+            }
+
+            info("----");
+            info("Min cache size [min=" + min + ", idx=" + minIdx + ']');
+            info("Current cache size " + cache.size());
+            info("Current cache key size " + cache.size());
+
+            check(max, PUT_ENTRY_SIZE);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @param arr Array.
+     * @param idx Index.
+     * @return Entry at the index.
+     */
+    protected MockEntry entry(MockEntry[] arr, int idx) {
+        MockEntry e = arr[idx];
+
+        if (e.isEvicted())
+            e = arr[idx] = new MockEntry(e.getKey());
+
+        return e;
+    }
+
+    /**
+     * @param prefix Prefix.
+     * @param p Policy.
+     */
+    protected void info(String prefix, EvictionPolicy<?, ?> p) {
+        info(prefix + ": " + p.toString());
+    }
+
+    /** @param p Policy. */
+    protected void info(EvictionPolicy<?, ?> p) {
+        info(p.toString());
+    }
+
+    /**
+     * @param c1 Policy collection.
+     * @param c2 Expected list.
+     */
+    protected static void check(Collection<EvictableEntry<String, String>> c1, MockEntry... c2) {
+        check(c1, F.asList(c2));
+    }
+
+    /**
+     * @param expSize Expected size.
+     * @param entrySize Entry size.
+     */
+    protected void check(int expSize, int entrySize) {
+        EvictionPolicyProxy proxy = proxy(policy());
+
+        assertEquals(expSize, proxy.getCurrentSize());
+        assertEquals(expSize * entrySize, proxy.getCurrentMemorySize());
+    }
+
+    /**
+     * @param entrySize Entry size.
+     * @param c1 Closure 1.
+     * @param c2 Closure 2.
+     */
+    protected void check(int entrySize, Collection<EvictableEntry<String, String>> c1, MockEntry... c2) {
+        check(c2.length, entrySize);
+
+        check(c1, c2);
+    }
+
+    /** @return Policy. */
+    protected T policy() {
+        GridCacheEvictionManager evictMgr = grid().cachex(DEFAULT_CACHE_NAME).context().evicts();
+
+        assert evictMgr != null;
+
+        return (T)evictMgr.getEvictionPolicy();
+    }
+
+    /**
+     * @param i Grid index.
+     * @return Policy.
+     */
+    @SuppressWarnings({"unchecked"})
+    protected T policy(int i) {
+        GridCacheEvictionManager evictMgr = grid(i).cachex(DEFAULT_CACHE_NAME).context().evicts();
+
+        assert evictMgr != null;
+
+        return (T)evictMgr.getEvictionPolicy();
+    }
+
+    /**
+     * @param i Grid index.
+     * @return Policy.
+     */
+    @SuppressWarnings({"unchecked"})
+    protected T nearPolicy(int i) {
+        GridCacheEvictionManager evictMgr = grid(i).cachex(DEFAULT_CACHE_NAME).context().near().context().evicts();
+
+        assert evictMgr  !=null;
+
+        return (T)evictMgr.getEvictionPolicy();
+    }
+
+    /**
+     * @param c1 Policy collection.
+     * @param c2 Expected list.
+     */
+    protected static void check(Collection<EvictableEntry<String, String>> c1, List<MockEntry> c2) {
+        assert c1.size() == c2.size() : "Mismatch [actual=" + string(c1) + ", expected=" + string(c2) + ']';
+
+        assert c1.containsAll(c2) : "Mismatch [actual=" + string(c1) + ", expected=" + string(c2) + ']';
+
+        int i = 0;
+
+        // Check order.
+        for (Cache.Entry<String, String> e : c1)
+            assertEquals(e, c2.get(i++));
+    }
+
+    /**
+     * @param c Collection.
+     * @return String.
+     */
+    @SuppressWarnings("unchecked")
+    protected static String string(Iterable<? extends Cache.Entry> c) {
+        return "[" +
+            F.fold(
+                c,
+                "",
+                new C2<Cache.Entry, String, String>() {
+                    @Override public String apply(Cache.Entry e, String b) {
+                        return b.isEmpty() ? e.getKey().toString() : b + ", " + e.getKey();
+                    }
+                }) +
+            "]]";
+    }
+
+    /** @throws Exception If failed. */
+    public void testMaxSizePartitionedNearDisabled() throws Exception {
+        mode = PARTITIONED;
+        nearEnabled = false;
+        plcMax = 10;
+        syncCommit = true;
+
+        gridCnt = 2;
+
+        checkPartitioned();
+    }
+
+    /** @throws Exception If failed. */
+    public void testMaxSizePartitionedNearDisabledWithBatch() throws Exception {
+        mode = PARTITIONED;
+        nearEnabled = false;
+        plcMax = 10;
+        plcBatchSize = 2;
+        syncCommit = true;
+
+        gridCnt = 2;
+
+        checkPartitioned();
+    }
+
+    /** @throws Exception If failed. */
+    public void testMaxMemSizePartitionedNearDisabled() throws Exception {
+        mode = PARTITIONED;
+        nearEnabled = false;
+        plcMax = 0;
+        plcMaxMemSize = 100;
+        syncCommit = true;
+
+        gridCnt = 2;
+
+        checkPartitioned();
+    }
+
+    /** @throws Exception If failed. */
+    public void testPartitionedNearEnabled() throws Exception {
+        mode = PARTITIONED;
+        nearEnabled = true;
+        nearMax = 3;
+        plcMax = 10;
+        syncCommit = true;
+
+        gridCnt = 2;
+
+        checkPartitioned(); // Near size is 0 because of backups present.
+    }
+
+    /** @throws Exception If failed. */
+    public void testPartitionedNearDisabledMultiThreaded() throws Exception {
+        mode = PARTITIONED;
+        nearEnabled = false;
+        plcMax = 100;
+
+        gridCnt = 2;
+
+        checkPartitionedMultiThreaded();
+    }
+
+    /** @throws Exception If failed. */
+    public void testPartitionedNearEnabledMultiThreaded() throws Exception {
+        mode = PARTITIONED;
+        nearEnabled = true;
+        plcMax = 10;
+
+        gridCnt = 2;
+
+        checkPartitionedMultiThreaded();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    protected void checkPartitioned() throws Exception {
+        int endSize = nearEnabled ? 0 : plcMax;
+
+        int endPlcSize = nearEnabled ? 0 : plcMax;
+
+        policyFactory = createPolicyFactory();
+
+        startGridsMultiThreaded(gridCnt);
+
+        try {
+            Random rand = new Random();
+
+            int cnt = 500;
+
+            for (int i = 0; i < cnt; i++) {
+                IgniteCache<Integer, String> cache = grid(rand.nextInt(2)).cache(DEFAULT_CACHE_NAME);
+
+                int key = rand.nextInt(100);
+                String val = Integer.toString(key);
+
+                cache.put(key, val);
+
+                if (i % 100 == 0)
+                    info("Stored cache object for key [key=" + key + ", idx=" + i + ']');
+            }
+
+            if (nearEnabled) {
+                for (int i = 0; i < gridCnt; i++)
+                    assertEquals(endSize, near(i).nearSize());
+
+                if (endPlcSize >= 0)
+                    checkNearPolicies(endPlcSize);
+            }
+            else {
+                if (plcMaxMemSize > 0) {
+                    for (int i = 0; i < gridCnt; i++) {
+                        GridDhtColocatedCache<Object, Object> cache = colocated(i);
+
+                        int memSize = 0;
+
+                        for (Cache.Entry<Object, Object> entry : cache.entrySet())
+                            memSize += entry.unwrap(EvictableEntry.class).size();
+
+                        EvictionPolicyProxy plc = proxy(policy(i));
+
+                        assertTrue(plc.getCurrentMemorySize() <= memSize);
+                    }
+                }
+
+                if (plcMax > 0) {
+                    for (int i = 0; i < gridCnt; i++) {
+                        int actual = colocated(i).size();
+
+                        assertTrue("Cache size is greater then policy size [expected=" + endSize + ", actual=" + actual + ']',
+                            actual <= endSize + (plcMaxMemSize > 0 ? 1 : plcBatchSize));
+                    }
+                }
+
+                checkPolicies();
+            }
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    protected void checkPartitionedMultiThreaded() throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGridsMultiThreaded(gridCnt);
+
+            final Random rand = new Random();
+
+            final AtomicInteger cntr = new AtomicInteger();
+
+            multithreaded(new Callable() {
+                @Nullable @Override public Object call() throws Exception {
+                    int cnt = 100;
+
+                    for (int i = 0; i < cnt && !Thread.currentThread().isInterrupted(); i++) {
+                        IgniteEx grid = grid(rand.nextInt(2));
+
+                        IgniteCache<Integer, String> cache = grid.cache(DEFAULT_CACHE_NAME);
+
+                        int key = rand.nextInt(1000);
+                        String val = Integer.toString(key);
+
+                        try (Transaction tx = grid.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                            String v = cache.get(key);
+
+                            assert v == null || v.equals(Integer.toString(key)) : "Invalid value for key [key=" + key +
+                                ", val=" + v + ']';
+
+                            cache.put(key, val);
+
+                            tx.commit();
+                        }
+
+                        if (cntr.incrementAndGet() % 100 == 0)
+                            info("Stored cache object for key [key=" + key + ", idx=" + i + ']');
+                    }
+
+                    return null;
+                }
+            }, 10);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @return Policy.
+     * @deprecated replace with getPolicyFactory();
+     */
+    @Deprecated
+    protected T createPolicy() {
+        return null;
+    }
+
+    /**
+     * @return Policy.
+     */
+    protected abstract Factory<T> createPolicyFactory();
+
+    /**
+     * @param nearMax Near max.
+     * @return Policy.
+     */
+    protected abstract Factory<T> createNearPolicyFactory(int nearMax);
+
+    /**
+     * Performs after-test near policy check.
+     *
+     * @param nearMax Near max.
+     */
+    protected void checkNearPolicies(int nearMax) {
+        for (int i = 0; i < gridCnt; i++) {
+
+            EvictionPolicyProxy proxy = proxy(nearPolicy(i));
+
+            for (EvictableEntry e : proxy.queue())
+                assert !e.isCached() : "Invalid near policy size: " + proxy.queue();
+        }
+    }
+
+    /**
+     * Performs after-test policy check.
+     */
+    protected void checkPolicies() {
+        for (int i = 0; i < gridCnt; i++) {
+            if (plcMaxMemSize > 0) {
+                int size = 0;
+
+                for (EvictableEntry entry : proxy(policy(i)).queue())
+                    size += entry.size();
+
+                assertEquals(size, proxy(policy(i)).getCurrentMemorySize());
+            }
+            else
+                assertTrue(proxy(policy(i)).queue().size() <= plcMax + plcBatchSize);
+        }
+    }
+
+    /**
+     *
+     */
+    @SuppressWarnings({"PublicConstructorInNonPublicClass"})
+    protected static class MockEntry extends GridCacheMockEntry<String, String> {
+        /** Key size. */
+        public static final int KEY_SIZE = 1;
+
+        /** Value size. */
+        public static final int VALUE_SIZE = 1;
+
+        /** Entry size. */
+        public static final int ENTRY_SIZE = KEY_SIZE + VALUE_SIZE;
+
+        /** */
+        private IgniteCache<String, String> parent;
+
+        /** Entry value. */
+        private String val;
+
+        /** @param key Key. */
+        public MockEntry(String key) {
+            super(key);
+        }
+
+        /**
+         * @param key Key.
+         * @param val Value.
+         */
+        public MockEntry(String key, String val) {
+            super(key);
+
+            this.val = val;
+        }
+
+        /**
+         * @param key Key.
+         * @param parent Parent.
+         */
+        public MockEntry(String key, @Nullable IgniteCache<String, String> parent) {
+            super(key);
+
+            this.parent = parent;
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("unchecked")
+        @Override public <T> T unwrap(Class<T> clazz) {
+            if (clazz.isAssignableFrom(IgniteCache.class))
+                return (T)parent;
+
+            return super.unwrap(clazz);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getValue() throws IllegalStateException {
+            return val;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int size() {
+            return val == null ? KEY_SIZE : ENTRY_SIZE;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(MockEntry.class, this, super.toString());
+        }
+    }
+
+    /**
+     * Rvicition policy proxy.
+     */
+    public static class EvictionPolicyProxy implements EvictionPolicy {
+        /** Policy. */
+        private final EvictionPolicy plc;
+
+        /**
+         * @param plc Policy.
+         */
+        private EvictionPolicyProxy(EvictionPolicy plc) {
+            this.plc = plc;
+        }
+
+        /**
+         * @param plc Policy.
+         * @return Policy proxy.
+         */
+        public static EvictionPolicyProxy proxy(EvictionPolicy plc) {
+            return new EvictionPolicyProxy(plc);
+        }
+
+        /**
+         * @return Get current size.
+         */
+        int getCurrentSize() {
+            try {
+                return (Integer)plc.getClass().getDeclaredMethod("getCurrentSize").invoke(plc);
+            }
+            catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        /**
+         * @return Current memory size.
+         */
+        long getCurrentMemorySize() {
+            try {
+                return (Long)plc.getClass().getMethod("getCurrentMemorySize").invoke(plc);
+            }
+            catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        /**
+         * @return Current queue.
+         */
+        public Collection<EvictableEntry> queue() {
+            try {
+                return (Collection<EvictableEntry>)plc.getClass().getDeclaredMethod("queue").invoke(plc);
+            }
+            catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        /**
+         * @param rmv Remove.
+         * @param entry Entry.
+         */
+        @Override public void onEntryAccessed(boolean rmv, EvictableEntry entry) {
+            try {
+                plc.getClass()
+                    .getMethod("onEntryAccessed", boolean.class, EvictableEntry.class)
+                    .invoke(plc, rmv, entry);
+            }
+            catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/FifoEvictionPolicyFactorySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/FifoEvictionPolicyFactorySelfTest.java
new file mode 100644
index 0000000000000..472bf41949091
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/FifoEvictionPolicyFactorySelfTest.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.eviction.fifo;
+
+import javax.cache.configuration.Factory;
+import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy;
+import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicyFactory;
+import org.apache.ignite.internal.processors.cache.eviction.EvictionPolicyFactoryAbstractTest;
+
+/**
+ * FIFO eviction policy tests.
+ */
+public class FifoEvictionPolicyFactorySelfTest extends EvictionPolicyFactoryAbstractTest<FifoEvictionPolicy<String, String>> {
+       /** {@inheritDoc} */
+    @Override protected Factory<FifoEvictionPolicy<String, String>> createPolicyFactory() {
+        return new FifoEvictionPolicyFactory<>(plcMax, plcBatchSize, plcMaxMemSize);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Factory<FifoEvictionPolicy<String, String>> createNearPolicyFactory(int nearMax) {
+        FifoEvictionPolicyFactory<String, String> plc = new FifoEvictionPolicyFactory<>();
+
+        plc.setMaxSize(nearMax);
+        plc.setBatchSize(plcBatchSize);
+
+        return plc;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void doTestPolicy() throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGrid();
+
+            MockEntry e1 = new MockEntry("1", "1");
+            MockEntry e2 = new MockEntry("2", "2");
+            MockEntry e3 = new MockEntry("3", "3");
+            MockEntry e4 = new MockEntry("4", "4");
+            MockEntry e5 = new MockEntry("5", "5");
+
+            FifoEvictionPolicy<String, String> p = policy();
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1);
+
+            p.onEntryAccessed(false, e2);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2);
+
+            p.onEntryAccessed(false, e3);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2, e3);
+
+            assert !e1.isEvicted();
+            assert !e2.isEvicted();
+            assert !e3.isEvicted();
+
+            p.onEntryAccessed(false, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e2, e3, e4);
+
+            assert e1.isEvicted();
+            assert !e2.isEvicted();
+            assert !e3.isEvicted();
+            assert !e4.isEvicted();
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assert e2.isEvicted();
+            assert !e3.isEvicted();
+            assert !e4.isEvicted();
+            assert !e5.isEvicted();
+
+            p.onEntryAccessed(false, e1 = new MockEntry("1", "1"));
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e5, e1);
+
+            assert e3.isEvicted();
+            assert !e1.isEvicted();
+            assert !e4.isEvicted();
+            assert !e5.isEvicted();
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e5, e1);
+
+            assert !e1.isEvicted();
+            assert !e4.isEvicted();
+            assert !e5.isEvicted();
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e5, e1);
+
+            assert !e1.isEvicted();
+            assert !e4.isEvicted();
+            assert !e5.isEvicted();
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e5, e1);
+
+            assert !e1.isEvicted();
+            assert !e4.isEvicted();
+            assert !e5.isEvicted();
+
+            p.onEntryAccessed(true, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e5);
+
+            assert !e1.isEvicted();
+            assert !e4.isEvicted();
+            assert !e5.isEvicted();
+
+            p.onEntryAccessed(true, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e5);
+
+            assert !e4.isEvicted();
+            assert !e5.isEvicted();
+
+            p.onEntryAccessed(true, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue());
+
+            assert !e5.isEvicted();
+
+            info(p);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void doTestPolicyWithBatch() throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGrid();
+
+            MockEntry e1 = new MockEntry("1", "1");
+            MockEntry e2 = new MockEntry("2", "2");
+            MockEntry e3 = new MockEntry("3", "3");
+            MockEntry e4 = new MockEntry("4", "4");
+            MockEntry e5 = new MockEntry("5", "5");
+
+            FifoEvictionPolicy<String, String> p = policy();
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1);
+
+            p.onEntryAccessed(false, e2);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2);
+
+            p.onEntryAccessed(false, e3);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2, e3);
+
+            p.onEntryAccessed(false, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2, e3, e4);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            // Batch evicted.
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertTrue(e1.isEvicted());
+            assertTrue(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e1 = new MockEntry("1", "1"));
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5, e1);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+            assertFalse(e1.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5, e1);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+            assertFalse(e1.isEvicted());
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5, e1);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+            assertFalse(e1.isEvicted());
+
+            p.onEntryAccessed(true, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3);
+
+            assertFalse(e3.isEvicted());
+
+            p.onEntryAccessed(true, e3);
+
+            check(MockEntry.ENTRY_SIZE, p.queue());
+
+            assertFalse(e3.isEvicted());
+
+            info(p);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruEvictionPolicyFactorySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruEvictionPolicyFactorySelfTest.java
new file mode 100644
index 0000000000000..d53cb6f36c923
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruEvictionPolicyFactorySelfTest.java
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.eviction.lru;
+
+import javax.cache.configuration.Factory;
+import org.apache.ignite.cache.eviction.EvictableEntry;
+import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy;
+import org.apache.ignite.cache.eviction.lru.LruEvictionPolicyFactory;
+import org.apache.ignite.internal.processors.cache.eviction.EvictionPolicyFactoryAbstractTest;
+
+/**
+ * LRU Eviction policy tests.
+ */
+public class LruEvictionPolicyFactorySelfTest extends EvictionPolicyFactoryAbstractTest<LruEvictionPolicy<String, String>> {
+    /** {@inheritDoc} */
+    @Override protected Factory<LruEvictionPolicy<String, String>> createPolicyFactory() {
+        return new LruEvictionPolicyFactory<>(plcMax, plcBatchSize, plcMaxMemSize);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Factory<LruEvictionPolicy<String, String>> createNearPolicyFactory(int nearMax) {
+        LruEvictionPolicyFactory<String, String> plc = new LruEvictionPolicyFactory<>();
+
+        plc.setMaxSize(nearMax);
+        plc.setBatchSize(plcBatchSize);
+
+        return plc;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMiddleAccess() throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGrid();
+
+            LruEvictionPolicy<String, String> p = policy();
+
+            int max = 8;
+
+            p.setMaxSize(max * MockEntry.ENTRY_SIZE);
+
+            MockEntry entry1 = new MockEntry("1", "1");
+            MockEntry entry2 = new MockEntry("2", "2");
+            MockEntry entry3 = new MockEntry("3", "3");
+
+            p.onEntryAccessed(false, entry1);
+            p.onEntryAccessed(false, entry2);
+            p.onEntryAccessed(false, entry3);
+
+            MockEntry[] freqUsed = new MockEntry[] {
+                new MockEntry("4", "4"),
+                new MockEntry("5", "5"),
+                new MockEntry("6", "6"),
+                new MockEntry("7", "7"),
+                new MockEntry("8", "7")
+            };
+
+            for (MockEntry e : freqUsed)
+                p.onEntryAccessed(false, e);
+
+            for (MockEntry e : freqUsed)
+                assert !e.isEvicted();
+
+            int cnt = 1001;
+
+            for (int i = 0; i < cnt; i++)
+                p.onEntryAccessed(false, entry(freqUsed, i % freqUsed.length));
+
+            info(p);
+
+            check(max, MockEntry.ENTRY_SIZE);
+        }
+        finally {
+            stopGrid();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void doTestPolicy() throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGrid();
+
+            MockEntry e1 = new MockEntry("1", "1");
+            MockEntry e2 = new MockEntry("2", "2");
+            MockEntry e3 = new MockEntry("3", "3");
+            MockEntry e4 = new MockEntry("4", "4");
+            MockEntry e5 = new MockEntry("5", "5");
+
+            LruEvictionPolicy<String, String> p = policy();
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1);
+
+            p.onEntryAccessed(false, e2);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2);
+
+            p.onEntryAccessed(false, e3);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2, e3);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+
+            p.onEntryAccessed(false, e4);
+
+            check(p.queue(), e2, e3, e4);
+            check(MockEntry.ENTRY_SIZE, p.queue(), e2, e3, e4);
+
+            assertTrue(e1.isEvicted());
+            assertFalse(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertTrue(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e1 = new MockEntry("1", "1"));
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e5, e1);
+
+            assertTrue(e3.isEvicted());
+            assertFalse(e1.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            assertEquals(3, p.getCurrentSize());
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e1, e5);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e5, e1);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            assertEquals(3, p.getCurrentSize());
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e1, e5);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e5);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e5);
+
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue());
+
+            assertFalse(e5.isEvicted());
+
+            info(p);
+        }
+        finally {
+            stopGrid();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void doTestPolicyWithBatch() throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGrid();
+
+            MockEntry e1 = new MockEntry("1", "1");
+            MockEntry e2 = new MockEntry("2", "2");
+            MockEntry e3 = new MockEntry("3", "3");
+            MockEntry e4 = new MockEntry("4", "4");
+            MockEntry e5 = new MockEntry("5", "5");
+
+            LruEvictionPolicy<String, String> p = policy();
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1);
+
+            p.onEntryAccessed(false, e2);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2);
+
+            p.onEntryAccessed(false, e3);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2, e3);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+
+            p.onEntryAccessed(false, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2, e3, e4);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            // Batch evicted
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertTrue(e1.isEvicted());
+            assertTrue(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e1 = new MockEntry("1", "1"));
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5, e1);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+            assertFalse(e1.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e1, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e1.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5, e1);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+            assertFalse(e1.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e1, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e1.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3);
+
+            assertFalse(e3.isEvicted());
+
+            p.onEntryAccessed(true, e3);
+
+            check(MockEntry.ENTRY_SIZE, p.queue());
+
+            info(p);
+        }
+        finally {
+            stopGrid();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void checkNearPolicies(int endNearPlcSize) {
+        for (int i = 0; i < gridCnt; i++)
+            for (EvictableEntry<String, String> e : nearPolicy(i).queue())
+                assert !e.isCached() : "Invalid near policy size: " + nearPolicy(i).queue();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void checkPolicies() {
+        for (int i = 0; i < gridCnt; i++) {
+            if (plcMaxMemSize > 0) {
+                int size = 0;
+
+                for (EvictableEntry<String, String> entry : policy(i).queue())
+                    size += entry.size();
+
+                assertEquals(size, policy(i).getCurrentMemorySize());
+            }
+            else
+                assertTrue(policy(i).queue().size() <= plcMax + plcBatchSize);
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/sorted/SortedEvictionPolicyFactorySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/sorted/SortedEvictionPolicyFactorySelfTest.java
new file mode 100644
index 0000000000000..a0ab18f0204db
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/sorted/SortedEvictionPolicyFactorySelfTest.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.eviction.sorted;
+
+import javax.cache.configuration.Factory;
+import org.apache.ignite.cache.eviction.sorted.SortedEvictionPolicy;
+import org.apache.ignite.cache.eviction.sorted.SortedEvictionPolicyFactory;
+import org.apache.ignite.internal.processors.cache.eviction.EvictionPolicyFactoryAbstractTest;
+
+/**
+ * Sorted eviction policy tests.
+ */
+public class SortedEvictionPolicyFactorySelfTest extends EvictionPolicyFactoryAbstractTest<SortedEvictionPolicy<String, String>> {
+    /** {@inheritDoc} */
+    @Override protected Factory<SortedEvictionPolicy<String, String>> createPolicyFactory() {
+        return new SortedEvictionPolicyFactory<>(plcMax, plcBatchSize, plcMaxMemSize);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Factory<SortedEvictionPolicy<String, String>> createNearPolicyFactory(int nearMax) {
+        SortedEvictionPolicyFactory<String, String> plc = new SortedEvictionPolicyFactory<>();
+
+        plc.setMaxSize(nearMax);
+        plc.setBatchSize(plcBatchSize);
+
+        return plc;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void doTestPolicy() throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGrid();
+
+            MockEntry e1 = new MockEntry("1", "1");
+            MockEntry e2 = new MockEntry("2", "2");
+            MockEntry e3 = new MockEntry("3", "3");
+            MockEntry e4 = new MockEntry("4", "4");
+            MockEntry e5 = new MockEntry("5", "5");
+
+            SortedEvictionPolicy<String, String> p = policy();
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1);
+
+            p.onEntryAccessed(false, e2);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2);
+
+            p.onEntryAccessed(false, e3);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2, e3);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2, e3);
+
+            p.onEntryAccessed(false, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e2, e3, e4);
+
+            assertTrue(e1.isEvicted());
+            assertFalse(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertTrue(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e1 = new MockEntry("1", "1"));
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertTrue(e1.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertTrue(e1.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e3);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e4, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e5);
+
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue());
+
+            assertFalse(e5.isEvicted());
+
+            info(p);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void doTestPolicyWithBatch() throws Exception {
+        policyFactory = createPolicyFactory();
+
+        try {
+            startGrid();
+
+            MockEntry e1 = new MockEntry("1", "1");
+            MockEntry e2 = new MockEntry("2", "2");
+            MockEntry e3 = new MockEntry("3", "3");
+            MockEntry e4 = new MockEntry("4", "4");
+            MockEntry e5 = new MockEntry("5", "5");
+
+            SortedEvictionPolicy<String, String> p = policy();
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1);
+
+            p.onEntryAccessed(false, e2);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2);
+
+            p.onEntryAccessed(false, e3);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2, e3);
+
+            p.onEntryAccessed(false, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e2, e3, e4);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            // Batch evicted.
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertTrue(e1.isEvicted());
+            assertTrue(e2.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e1 = new MockEntry("1", "1"));
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e3, e4, e5);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e3, e4, e5);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(false, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e1, e3, e4, e5);
+
+            assertFalse(e1.isEvicted());
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e1);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e4, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e4.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e4);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3, e5);
+
+            assertFalse(e3.isEvicted());
+            assertFalse(e5.isEvicted());
+
+            p.onEntryAccessed(true, e5);
+
+            check(MockEntry.ENTRY_SIZE, p.queue(), e3);
+
+            assertFalse(e3.isEvicted());
+
+            p.onEntryAccessed(true, e3);
+
+            check(MockEntry.ENTRY_SIZE, p.queue());
+
+            assertFalse(e3.isEvicted());
+
+            info(p);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java
index 1ad63ee13ff56..fa1eac12fcfc5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java
@@ -33,10 +33,13 @@
 import org.apache.ignite.internal.processors.cache.eviction.GridCacheEvictionLockUnlockSelfTest;
 import org.apache.ignite.internal.processors.cache.eviction.GridCacheEvictionTouchSelfTest;
 import org.apache.ignite.internal.processors.cache.eviction.GridCacheSynchronousEvictionsFailoverSelfTest;
+import org.apache.ignite.internal.processors.cache.eviction.fifo.FifoEvictionPolicyFactorySelfTest;
 import org.apache.ignite.internal.processors.cache.eviction.fifo.FifoEvictionPolicySelfTest;
+import org.apache.ignite.internal.processors.cache.eviction.lru.LruEvictionPolicyFactorySelfTest;
 import org.apache.ignite.internal.processors.cache.eviction.lru.LruEvictionPolicySelfTest;
 import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearEvictionPolicySelfTest;
 import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearOnlyNearEvictionPolicySelfTest;
+import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicyFactorySelfTest;
 import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicySelfTest;
 
 /**
@@ -53,6 +56,9 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(FifoEvictionPolicySelfTest.class));
         suite.addTest(new TestSuite(SortedEvictionPolicySelfTest.class));
         suite.addTest(new TestSuite(LruEvictionPolicySelfTest.class));
+        suite.addTest(new TestSuite(FifoEvictionPolicyFactorySelfTest.class));
+        suite.addTest(new TestSuite(SortedEvictionPolicyFactorySelfTest.class));
+        suite.addTest(new TestSuite(LruEvictionPolicyFactorySelfTest.class));
         suite.addTest(new TestSuite(LruNearEvictionPolicySelfTest.class));
         suite.addTest(new TestSuite(LruNearOnlyNearEvictionPolicySelfTest.class));
         suite.addTest(new TestSuite(GridCacheNearEvictionSelfTest.class));
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteNode.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteNode.java
index a261b98514b78..c8d2b49f25283 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteNode.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteNode.java
@@ -101,8 +101,10 @@ public IgniteNode(boolean clientMode, Ignite ignite) {
                 if (args.isNearCache()) {
                     NearCacheConfiguration nearCfg = new NearCacheConfiguration();
 
-                    if (args.getNearCacheSize() != 0)
-                        nearCfg.setNearEvictionPolicy(new LruEvictionPolicy(args.getNearCacheSize()));
+                    int nearCacheSize = args.getNearCacheSize();
+
+                    if (nearCacheSize != 0)
+                        nearCfg.setNearEvictionPolicy(new LruEvictionPolicy(nearCacheSize));
 
                     cc.setNearConfiguration(nearCfg);
                 }

From c2feb91cf801fee28bc77019b46c514975a28d8d Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Fri, 10 Nov 2017 23:05:15 +0300
Subject: [PATCH 426/516] GG-12915 DataStreamer.addData(single entry) produces
 too much garbage

---
 .../util/future/GridFutureAdapter.java        | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
index 257f1996ddd08..925d77c8ef849 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.internal.util.future;
 
 import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.AbstractQueuedSynchronizer;
@@ -213,10 +215,10 @@ private R get0(boolean ignoreInterrupts) throws IgniteCheckedException {
                 if (!done) {
                     if (lsnr == null)
                         lsnr = lsnr0;
-                    else if (lsnr instanceof ArrayListener)
-                        ((ArrayListener)lsnr).add(lsnr0);
+                    else if (lsnr instanceof GridFutureAdapter.LinkedListListener)
+                        ((LinkedListListener)lsnr).add(lsnr0);
                     else
-                        lsnr = (IgniteInClosure)new ArrayListener<IgniteInternalFuture>(lsnr, lsnr0);
+                        lsnr = (IgniteInClosure)new LinkedListListener<IgniteInternalFuture>(lsnr, lsnr0);
 
                     return;
                 }
@@ -436,38 +438,39 @@ private String state() {
     /**
      *
      */
-    private static class ArrayListener<R> implements IgniteInClosure<IgniteInternalFuture<R>> {
+    private static class LinkedListListener<R> implements IgniteInClosure<IgniteInternalFuture<R>> {
         /** */
         private static final long serialVersionUID = 0L;
 
         /** */
-        private IgniteInClosure<? super IgniteInternalFuture<R>>[] arr;
+        private List<IgniteInClosure<? super IgniteInternalFuture<R>>> list;
 
         /**
          * @param lsnrs Listeners.
          */
-        private ArrayListener(IgniteInClosure... lsnrs) {
-            this.arr = lsnrs;
+        private LinkedListListener(IgniteInClosure... lsnrs) {
+            list = new LinkedList<>();
+            for (IgniteInClosure lsnr : lsnrs) {
+                list.add(lsnr);
+            }
         }
 
         /** {@inheritDoc} */
         @Override public void apply(IgniteInternalFuture<R> fut) {
-            for (int i = 0; i < arr.length; i++)
-                arr[i].apply(fut);
+            for (IgniteInClosure<? super IgniteInternalFuture<R>> closure : list)
+                closure.apply(fut);
         }
 
         /**
          * @param lsnr Listener.
          */
         void add(IgniteInClosure<? super IgniteInternalFuture<R>> lsnr) {
-            arr = Arrays.copyOf(arr, arr.length + 1);
-
-            arr[arr.length - 1] = lsnr;
+            list.add(lsnr);
         }
 
         /** {@inheritDoc} */
         @Override public String toString() {
-            return S.toString(ArrayListener.class, this, "arrSize", arr.length);
+            return S.toString(LinkedListListener.class, this, "listSize", list.size());
         }
     }
 

From ebf512c02c05dd1644207a1b7359c1c8319bf297 Mon Sep 17 00:00:00 2001
From: amashenkov <andrey.mashenkov@gmail.com>
Date: Mon, 13 Nov 2017 10:26:53 +0300
Subject: [PATCH 427/516] GG-13021: Fixed NPE on node stop when SSL is used.

(cherry picked from commit 132ec3f)
---
 .../internal/util/nio/GridNioServer.java      |  8 ++++--
 .../IgniteCommunicationBalanceTest.java       | 13 +++++++++
 .../IgniteCommunicationSslBalanceTest.java    | 28 +++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite.java      |  2 ++
 4 files changed, 49 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationSslBalanceTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index a59adba9d16e7..c3208f9d9ec0b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -3024,8 +3024,12 @@ protected HeadFilter() {
 
                     GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl)ses;
 
-                    if (!ses0.procWrite.get() && ses0.procWrite.compareAndSet(false, true))
-                        ses0.worker().registerWrite(ses0);
+                    if (!ses0.procWrite.get() && ses0.procWrite.compareAndSet(false, true)) {
+                        GridNioWorker worker = ses0.worker();
+
+                        if (worker != null)
+                            worker.registerWrite(ses0);
+                    }
 
                     return null;
                 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
index 4271417554fa7..a913cb96bb26c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
@@ -72,9 +72,19 @@ public class IgniteCommunicationBalanceTest extends GridCommonAbstractTest {
 
         cfg.setClientMode(client);
 
+        if (sslEnabled())
+            cfg.setSslContextFactory(GridTestUtils.sslFactory());
+
         return cfg;
     }
 
+    /**
+     * @return {@code True} to enable SSL.
+     */
+    protected boolean sslEnabled() {
+        return false;
+    }
+
     /**
      * @return Value for {@link TcpCommunicationSpi#setUsePairedConnections(boolean)}.
      */
@@ -100,6 +110,9 @@ protected int connectionsPerNode() {
      * @throws Exception If failed.
      */
     public void testBalance1() throws Exception {
+        if (sslEnabled())
+            return;
+
         System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "5000");
 
         try {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationSslBalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationSslBalanceTest.java
new file mode 100644
index 0000000000000..68094e265577a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationSslBalanceTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+/**
+ *
+ */
+public class IgniteCommunicationSslBalanceTest extends IgniteCommunicationBalanceTest {
+    /** {@inheritDoc} */
+    @Override protected boolean sslEnabled() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 159bc78c38ec3..fd8d43fa46d7a 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -47,6 +47,7 @@
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceMultipleConnectionsTest;
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalancePairedConnectionsTest;
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceTest;
+import org.apache.ignite.internal.managers.communication.IgniteCommunicationSslBalanceTest;
 import org.apache.ignite.internal.managers.communication.IgniteIoTestMessagesTest;
 import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest;
 import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest;
@@ -351,6 +352,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(IgniteCommunicationBalanceTest.class);
         suite.addTestSuite(IgniteCommunicationBalancePairedConnectionsTest.class);
         suite.addTestSuite(IgniteCommunicationBalanceMultipleConnectionsTest.class);
+        suite.addTestSuite(IgniteCommunicationSslBalanceTest.class);
         suite.addTestSuite(IgniteIoTestMessagesTest.class);
 
         suite.addTestSuite(GridStoreLoadCacheTest.class);

From 111fd8ba8c2a9459a731c3c4fde8ccfcff39dcc8 Mon Sep 17 00:00:00 2001
From: Tikhonov Nikolay <tikhonovnicolay@gmail.com>
Date: Sat, 11 Nov 2017 12:44:33 +0300
Subject: [PATCH 428/516] Fixed .NET test compilation.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../Cache/Store/CacheStoreSessionTest.cs                        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs
index 8ab3b25c65b94..8118305af3591 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs
@@ -110,7 +110,7 @@ public void TestSession()
             }
 
             Assert.AreEqual(1, _dumps.Count);
-            ops = _dumps.First();
+            var ops = _dumps.First();
             Assert.AreEqual(3, ops.Count);
 
             Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.Write && Cache1.Equals(op.CacheName) && 1.Equals(op.Key) && 1.Equals(op.Value)));

From 40a51bb83cb400a1e42e57ad955693e41faa175d Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 13 Nov 2017 10:35:21 +0300
Subject: [PATCH 429/516] IGNITE-6818 Handle half open connection in
 communication.

(cherry picked from commit 191295d)
---
 .../tcp/TcpCommunicationSpi.java              |  39 +++--
 ...municationSpiHalfOpenedConnectionTest.java | 142 ++++++++++++++++++
 .../IgniteSpiCommunicationSelfTestSuite.java  |   2 +
 3 files changed, 169 insertions(+), 14 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiHalfOpenedConnectionTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index dffa181058c56..55e2b6b95af98 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -65,7 +65,6 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
-import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.GridConcurrentFactory;
 import org.apache.ignite.internal.util.GridSpinReadWriteLock;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -94,6 +93,7 @@
 import org.apache.ignite.internal.util.nio.ssl.BlockingSslHandler;
 import org.apache.ignite.internal.util.nio.ssl.GridNioSslFilter;
 import org.apache.ignite.internal.util.nio.ssl.GridSslMeta;
+import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
@@ -529,15 +529,7 @@ private void onFirstMessage(final GridNioSession ses, Message msg) {
                         if (c.failed) {
                             ses.send(new RecoveryLastReceivedMessage(-1));
 
-                            for (GridNioSession ses0 : nioSrvr.sessions()) {
-                                ConnectionKey key0 = ses0.meta(CONN_IDX_META);
-
-                                if (ses0.accepted() && key0 != null &&
-                                    key0.nodeId().equals(connKey.nodeId()) &&
-                                    key0.connectionIndex() == connKey.connectionIndex() &&
-                                    key0.connectCount() < connKey.connectCount())
-                                    ses0.close();
-                            }
+                            closeStaleConnections(connKey);
                         }
                     }
                 }
@@ -557,11 +549,13 @@ private void onFirstMessage(final GridNioSession ses, Message msg) {
                         if (oldClient instanceof GridTcpNioCommunicationClient) {
                             if (log.isInfoEnabled())
                                 log.info("Received incoming connection when already connected " +
-                                "to this node, rejecting [locNode=" + locNode.id() +
-                                ", rmtNode=" + sndId + ']');
+                                    "to this node, rejecting [locNode=" + locNode.id() +
+                                    ", rmtNode=" + sndId + ']');
 
                             ses.send(new RecoveryLastReceivedMessage(-1));
 
+                            closeStaleConnections(connKey);
+
                             return;
                         }
                         else {
@@ -589,11 +583,13 @@ private void onFirstMessage(final GridNioSession ses, Message msg) {
 
                                 if (log.isInfoEnabled())
                                     log.info("Received incoming connection when already connected " +
-                                    "to this node, rejecting [locNode=" + locNode.id() +
-                                    ", rmtNode=" + sndId + ']');
+                                        "to this node, rejecting [locNode=" + locNode.id() +
+                                        ", rmtNode=" + sndId + ']');
 
                                 ses.send(new RecoveryLastReceivedMessage(-1));
 
+                                closeStaleConnections(connKey);
+
                                 fut.onDone(oldClient);
 
                                 return;
@@ -647,6 +643,21 @@ private void onFirstMessage(final GridNioSession ses, Message msg) {
                 }
             }
 
+            /**
+             * @param connKey Connection key.
+             */
+            private void closeStaleConnections(ConnectionKey connKey) {
+                for (GridNioSession ses0 : nioSrvr.sessions()) {
+                    ConnectionKey key0 = ses0.meta(CONN_IDX_META);
+
+                    if (ses0.accepted() && key0 != null &&
+                        key0.nodeId().equals(connKey.nodeId()) &&
+                        key0.connectionIndex() == connKey.connectionIndex() &&
+                        key0.connectCount() < connKey.connectCount())
+                        ses0.close();
+                }
+            }
+
             @Override public void onMessage(GridNioSession ses, Message msg) {
                 ConnectionKey connKey = ses.meta(CONN_IDX_META);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiHalfOpenedConnectionTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiHalfOpenedConnectionTest.java
new file mode 100644
index 0000000000000..3e10f942c4459
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiHalfOpenedConnectionTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
+import org.apache.ignite.internal.util.nio.GridNioRecoveryDescriptor;
+import org.apache.ignite.internal.util.nio.GridNioServerListener;
+import org.apache.ignite.internal.util.nio.GridTcpNioCommunicationClient;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests case when connection is closed only for one side, when other is not notified.
+ */
+public class TcpCommunicationSpiHalfOpenedConnectionTest extends GridCommonAbstractTest {
+    /** Client spi. */
+    private TcpCommunicationSpi clientSpi;
+
+    /** Paired connections. */
+    private boolean pairedConnections;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (igniteInstanceName.contains("client")) {
+            cfg.setClientMode(true);
+
+            clientSpi = (TcpCommunicationSpi)cfg.getCommunicationSpi();
+        }
+
+        ((TcpCommunicationSpi)cfg.getCommunicationSpi()).setUsePairedConnections(pairedConnections);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnect() throws Exception {
+        pairedConnections = false;
+
+        checkReconnect();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectPaired() throws Exception {
+        pairedConnections = true;
+
+        checkReconnect();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkReconnect() throws Exception {
+        Ignite srv = startGrid("server");
+        Ignite client = startGrid("client");
+
+        UUID nodeId = srv.cluster().localNode().id();
+
+        System.out.println(">> Server ID: " + nodeId);
+
+        ClusterGroup srvGrp = client.cluster().forNodeId(nodeId);
+
+        System.out.println(">> Send job");
+
+        // Establish connection
+        client.compute(srvGrp).run(F.noop());
+
+        ConcurrentMap<UUID, GridCommunicationClient[]> clients = U.field(clientSpi, "clients");
+        ConcurrentMap<?, GridNioRecoveryDescriptor> recoveryDescs = U.field(clientSpi, "recoveryDescs");
+        ConcurrentMap<?, GridNioRecoveryDescriptor> outRecDescs = U.field(clientSpi, "outRecDescs");
+        ConcurrentMap<?, GridNioRecoveryDescriptor> inRecDescs = U.field(clientSpi, "inRecDescs");
+        GridNioServerListener<Message> lsnr = U.field(clientSpi, "srvLsnr");
+
+        Iterator<GridNioRecoveryDescriptor> it = F.concat(
+            recoveryDescs.values().iterator(),
+            outRecDescs.values().iterator(),
+            inRecDescs.values().iterator()
+        );
+
+        while (it.hasNext()) {
+            GridNioRecoveryDescriptor desc = it.next();
+
+            // Need to simulate connection close in GridNioServer as it
+            // releases descriptors on disconnect.
+            desc.release();
+        }
+
+        // Remove client to avoid calling close(), in that case server
+        // will close connection too, but we want to keep the server
+        // uninformed and force ping old connection.
+        GridCommunicationClient[] clients0 = clients.remove(nodeId);
+
+        for (GridCommunicationClient commClient : clients0)
+            lsnr.onDisconnected(((GridTcpNioCommunicationClient)commClient).session(), new IOException("Test exception"));
+
+        info(">> Removed client");
+
+        // Reestablish connection
+        client.compute(srvGrp).run(F.noop());
+
+        info(">> Sent second job");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 30_000;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
index 77de3fcc54999..8e96a3f223604 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.spi.communication.tcp.IgniteTcpCommunicationRecoveryAckClosureSelfTest;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiDropNodesTest;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiFaultyClientTest;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiHalfOpenedConnectionTest;
 
 /**
  * Test suite for all communication SPIs.
@@ -78,6 +79,7 @@ public static TestSuite suite() throws Exception {
 
         suite.addTest(new TestSuite(TcpCommunicationSpiFaultyClientTest.class));
         suite.addTest(new TestSuite(TcpCommunicationSpiDropNodesTest.class));
+        suite.addTest(new TestSuite(TcpCommunicationSpiHalfOpenedConnectionTest.class));
 
         return suite;
     }

From 86e3349c168bf11b45ef219e93c86adf42cb3a55 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Tue, 14 Nov 2017 15:40:41 +0300
Subject: [PATCH 430/516] GG-12821 IGNITE-4642: Added "enforceJoinOrder" flag
 to thick JDBC driver.

Also add detection of UPDATE/DELETE in executeQuery(), tangentially related to IGNITE-6326
---
 .../jdbc2/JdbcConnectionSelfTest.java         | 25 ++++++
 .../jdbc2/JdbcDeleteStatementSelfTest.java    | 30 +++++++
 .../org/apache/ignite/IgniteJdbcDriver.java   | 14 +++-
 .../ignite/internal/jdbc2/JdbcConnection.java | 12 +++
 .../ignite/internal/jdbc2/JdbcQueryTask.java  |  2 +-
 .../internal/jdbc2/JdbcQueryTaskV2.java       |  8 ++
 .../internal/jdbc2/JdbcQueryTaskV3.java       | 49 +++++++++++
 .../ignite/internal/jdbc2/JdbcResultSet.java  | 19 ++++-
 .../ignite/internal/jdbc2/JdbcStatement.java  | 82 ++++++++++++++-----
 9 files changed, 216 insertions(+), 25 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java
index 0bbdca86c67f9..e080027059ec8 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java
@@ -268,6 +268,31 @@ public void testClose() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSqlHints() throws Exception {
+        try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "enforceJoinOrder=true@"
+            + CFG_URL)) {
+            assertTrue(((JdbcConnection)conn).isEnforceJoinOrder());
+            assertFalse(((JdbcConnection)conn).isDistributedJoins());
+            assertFalse(((JdbcConnection)conn).isCollocatedQuery());
+        }
+
+        try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "distributedJoins=true@"
+            + CFG_URL)) {
+            assertFalse(((JdbcConnection)conn).isEnforceJoinOrder());
+            assertTrue(((JdbcConnection)conn).isDistributedJoins());
+            assertFalse(((JdbcConnection)conn).isCollocatedQuery());
+        }
+
+        try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "collocated=true@" + CFG_URL)) {
+            assertFalse(((JdbcConnection)conn).isEnforceJoinOrder());
+            assertFalse(((JdbcConnection)conn).isDistributedJoins());
+            assertTrue(((JdbcConnection)conn).isCollocatedQuery());
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDeleteStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDeleteStatementSelfTest.java
index d55c979f7e52b..55cc8b949dbce 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDeleteStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDeleteStatementSelfTest.java
@@ -17,9 +17,13 @@
 
 package org.apache.ignite.internal.jdbc2;
 
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.concurrent.Callable;
+import javax.cache.CacheException;
+import org.apache.ignite.testframework.GridTestUtils;
 
 /**
  *
@@ -46,4 +50,30 @@ public void testExecuteUpdate() throws SQLException {
         assertFalse(jcache(0).containsKey("p2"));
         assertTrue(jcache(0).containsKeys(new HashSet<Object>(Arrays.asList("p1", "p3"))));
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStatementTypeMismatchUpdate() throws Exception {
+        GridTestUtils.assertThrowsAnyCause(log,
+            new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    conn.createStatement().executeQuery("delete from person where id=1");
+
+                    return null;
+                }
+            },
+            CacheException.class,
+            "Given statement type does not match that declared by JDBC driver");
+
+        ResultSet rs = conn.createStatement().executeQuery("select age from person where id=1");
+
+        boolean next = rs.next();
+
+        assert next;
+
+        assert rs.getInt(1) == 25 : "The data must not be updated. " +
+            "Because update statement is executed via 'executeQuery' method." +
+            " Data [val=" + rs.getInt(1) + ']';
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
index 0ee41c0fc5d10..9d1489a01f988 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
@@ -109,6 +109,10 @@
  *         combination with {@code local} and/or {@code collocated} flags with {@code true} value or in case of querying
  *         of local cache. Default value is {@code false}.
  *     </li>
+ *     <li>
+ *         {@code enforceJoinOrder} - Sets flag to enforce join order of tables in the query. If set to {@code true}
+ *          query optimizer will not reorder tables in join. By default is {@code false}.
+ *     </li>
  * </ul>
  *
  * <h2 class="header">Configuration of Ignite Java client based connection</h2>
@@ -289,6 +293,9 @@ public class IgniteJdbcDriver implements Driver {
     /** Collocated parameter name. */
     private static final String PARAM_COLLOCATED = "collocated";
 
+    /** Parameter: enforce join order flag. */
+    public static final String PARAM_ENFORCE_JOIN_ORDER = "enforceJoinOrder";
+
     /** Distributed joins parameter name. */
     private static final String PARAM_DISTRIBUTED_JOINS = "distributedJoins";
 
@@ -349,6 +356,10 @@ public class IgniteJdbcDriver implements Driver {
     /** Whether DML streaming will overwrite existing cache entries. */
     public static final String PROP_STREAMING_ALLOW_OVERWRITE = PROP_PREFIX + PARAM_STREAMING_ALLOW_OVERWRITE;
 
+    /** Enforce join order property name. */
+    public static final String PROP_ENFORCE_JOIN_ORDER = PROP_PREFIX + PARAM_ENFORCE_JOIN_ORDER;
+
+
     /** Cache name property name. */
     public static final String PROP_CFG = PROP_PREFIX + "cfg";
 
@@ -415,7 +426,8 @@ public class IgniteJdbcDriver implements Driver {
             new PropertyInfo("Local", info.getProperty(PROP_LOCAL), ""),
             new PropertyInfo("Collocated", info.getProperty(PROP_COLLOCATED), ""),
             new PropertyInfo("Distributed Joins", info.getProperty(PROP_DISTRIBUTED_JOINS), ""),
-            new PropertyInfo("Transactions Allowed", info.getProperty(PROP_TX_ALLOWED), "")
+            new PropertyInfo("Transactions Allowed", info.getProperty(PROP_TX_ALLOWED), ""),
+            new PropertyInfo("Enforce Join Order", info.getProperty(PROP_ENFORCE_JOIN_ORDER), "")
         );
 
         if (info.getProperty(PROP_CFG) != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
index b2b0dcb312031..9008e751ac745 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
@@ -75,6 +75,7 @@
 import static org.apache.ignite.IgniteJdbcDriver.PROP_CFG;
 import static org.apache.ignite.IgniteJdbcDriver.PROP_COLLOCATED;
 import static org.apache.ignite.IgniteJdbcDriver.PROP_DISTRIBUTED_JOINS;
+import static org.apache.ignite.IgniteJdbcDriver.PROP_ENFORCE_JOIN_ORDER;
 import static org.apache.ignite.IgniteJdbcDriver.PROP_LOCAL;
 import static org.apache.ignite.IgniteJdbcDriver.PROP_NODE_ID;
 import static org.apache.ignite.IgniteJdbcDriver.PROP_TX_ALLOWED;
@@ -149,6 +150,9 @@ public class JdbcConnection implements Connection {
     /** Allow overwrites for duplicate keys on streamed {@code INSERT}s. */
     private final boolean streamAllowOverwrite;
 
+    /** Enforced join order flag. */
+    private boolean enforceJoinOrder;
+
     /** Statements. */
     final Set<JdbcStatement> statements = new HashSet<>();
 
@@ -170,6 +174,7 @@ public JdbcConnection(String url, Properties props) throws SQLException {
         collocatedQry = Boolean.parseBoolean(props.getProperty(PROP_COLLOCATED));
         distributedJoins = Boolean.parseBoolean(props.getProperty(PROP_DISTRIBUTED_JOINS));
         txAllowed = Boolean.parseBoolean(props.getProperty(PROP_TX_ALLOWED));
+        enforceJoinOrder = Boolean.parseBoolean(props.getProperty(PROP_ENFORCE_JOIN_ORDER));
 
         stream = Boolean.parseBoolean(props.getProperty(PROP_STREAMING));
         streamAllowOverwrite = Boolean.parseBoolean(props.getProperty(PROP_STREAMING_ALLOW_OVERWRITE));
@@ -789,6 +794,13 @@ boolean isDistributedJoins() {
         return distributedJoins;
     }
 
+    /**
+     * @return Enforce join order flag.
+     */
+    boolean isEnforceJoinOrder() {
+        return enforceJoinOrder;
+    }
+
     /**
      * Ensures that connection is not closed.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java
index bd6b0f2c74c98..0d1d602b5856a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java
@@ -151,7 +151,7 @@ public JdbcQueryTask(Ignite ignite, String cacheName, String sql,
                     throw new SQLException("Cache not found [cacheName=" + cacheName + ']');
             }
 
-            SqlFieldsQuery qry = new SqlFieldsQuery(sql).setArgs(args);
+            SqlFieldsQuery qry = new JdbcSqlFieldsQuery(sql, true).setArgs(args);
 
             qry.setPageSize(fetchSize);
             qry.setLocal(locQry);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java
index 61f152dd66000..d9c91d5d12805 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV2.java
@@ -160,6 +160,7 @@ public JdbcQueryTaskV2(Ignite ignite, String cacheName, String sql,
             qry.setLocal(locQry);
             qry.setCollocated(collocatedQry);
             qry.setDistributedJoins(distributedJoins);
+            qry.setEnforceJoinOrder(enforceJoinOrder());
 
             QueryCursorImpl<List<?>> qryCursor = (QueryCursorImpl<List<?>>)cache.withKeepBinary().query(qry);
 
@@ -211,6 +212,13 @@ else if (!loc && !CURSORS.replace(uuid, cursor, new Cursor(cursor.cursor, cursor
         return new QueryResult(uuid, finished, isQry, rows, cols, tbls, types);
     }
 
+    /**
+     * @return Enforce join order flag.
+     */
+    protected boolean enforceJoinOrder() {
+        return false;
+    }
+
     /**
      * Schedules removal of stored cursor in case of remote query execution.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java
new file mode 100644
index 0000000000000..5bb2454332167
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+
+/**
+ *
+ */
+class JdbcQueryTaskV3 extends JdbcQueryTaskV2 {
+    /** Serial version uid. */
+    private static final long serialVersionUID = 0L;
+
+    /** Enforce join order flag. */
+    private final boolean enforceJoinOrder;
+
+    /**
+     * {@inheritDoc}
+     * @param enforceJoinOrder Enforce joins order falg.
+     */
+    public JdbcQueryTaskV3(Ignite ignite, String cacheName, String sql,
+                           Boolean isQry, boolean loc, Object[] args, int fetchSize, UUID uuid,
+                           boolean locQry, boolean collocatedQry, boolean distributedJoins, boolean enforceJoinOrder) {
+        super(ignite, cacheName, sql, isQry, loc, args, fetchSize, uuid, locQry, collocatedQry, distributedJoins);
+
+        this.enforceJoinOrder = enforceJoinOrder;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean enforceJoinOrder() {
+        return enforceJoinOrder;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
index 187930ec4cc66..e398397d61441 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
@@ -92,7 +92,9 @@ public class JdbcResultSet implements ResultSet {
     private final boolean useNewQryTask;
 
     /**
-     * Creates new result set.
+     * Creates new result set with predefined fields.
+     * Result set created with this constructor will
+     * never execute remote tasks.
      *
      * @param uuid Query UUID.
      * @param stmt Statement.
@@ -100,6 +102,7 @@ public class JdbcResultSet implements ResultSet {
      * @param cols Column names.
      * @param types Types.
      * @param fields Fields.
+     * @param finished Result set finished flag (the last result set).
      */
     JdbcResultSet(@Nullable UUID uuid, JdbcStatement stmt, List<String> tbls, List<String> cols,
         List<String> types, Collection<List<?>> fields, boolean finished) {
@@ -185,8 +188,17 @@ else if (!finished) {
 
             if (useNewQryTask) {
                 // Connections from new clients send queries with new tasks, so we have to continue in the same manner
-                JdbcQueryTaskV2 qryTask = new JdbcQueryTaskV2(loc ? ignite : null, conn.cacheName(), null, true, loc, null,
-                    fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(), conn.isDistributedJoins());
+                JdbcQueryTaskV2 qryTask;
+                if (conn.isEnforceJoinOrder()) {
+                    qryTask = new JdbcQueryTaskV3(loc ? ignite : null, conn.cacheName(), null, true, loc, null,
+                        fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(),
+                        conn.isDistributedJoins(), true);
+                }
+                else {
+                    qryTask = new JdbcQueryTaskV2(loc ? ignite : null, conn.cacheName(), null, true, loc, null,
+                        fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(),
+                        conn.isDistributedJoins());
+                }
 
                 try {
                     JdbcQueryTaskV2.QueryResult res =
@@ -243,6 +255,7 @@ else if (!finished) {
     /**
      * Marks result set as closed.
      * If this result set is associated with locally executed query then query cursor will also closed.
+     * @throws SQLException On error.
      */
     void closeInternal() throws SQLException  {
         if (((JdbcConnection)stmt.getConnection()).nodeId() == null && uuid != null) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java
index 44db3757eee40..9752a536641c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java
@@ -108,28 +108,53 @@ public class JdbcStatement implements Statement {
 
         boolean loc = nodeId == null;
 
-        JdbcQueryTask qryTask = new JdbcQueryTask(loc ? ignite : null, conn.cacheName(), sql, loc, getArgs(),
-            fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(), conn.isDistributedJoins());
+        JdbcResultSet rs;
+        if (conn.isEnforceJoinOrder()) {
+            JdbcQueryTaskV3 qryTask = new JdbcQueryTaskV3(loc ? ignite : null, conn.cacheName(), sql, true,
+                loc, getArgs(), fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(),
+                conn.isDistributedJoins(), true);
 
-        try {
-            JdbcQueryTask.QueryResult res =
-                loc ? qryTask.call() : ignite.compute(ignite.cluster().forNodeId(nodeId)).call(qryTask);
+            try {
+                JdbcQueryTaskV3.QueryResult res =
+                    loc ? qryTask.call() : ignite.compute(ignite.cluster().forNodeId(nodeId)).call(qryTask);
 
-            JdbcResultSet rs = new JdbcResultSet(uuid, this, res.getTbls(), res.getCols(), res.getTypes(),
-                res.getRows(), res.isFinished());
+                rs = new JdbcResultSet(uuid, this, res.getTbls(), res.getCols(), res.getTypes(),
+                    res.getRows(), res.isFinished());
 
-            rs.setFetchSize(fetchSize);
+                rs.setFetchSize(fetchSize);
+            }
+            catch (IgniteSQLException e) {
+                throw e.toJdbcException();
+            }
+            catch (Exception e) {
+                throw new SQLException("Failed to query Ignite.", e);
+            }
+        }
+        else {
+            JdbcQueryTask qryTask = new JdbcQueryTask(loc ? ignite : null, conn.cacheName(), sql, loc, getArgs(),
+                fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(),
+                conn.isDistributedJoins());
 
-            resSets.add(rs);
+            try {
+                JdbcQueryTask.QueryResult res =
+                    loc ? qryTask.call() : ignite.compute(ignite.cluster().forNodeId(nodeId)).call(qryTask);
 
-            return rs;
-        }
-        catch (IgniteSQLException e) {
-            throw e.toJdbcException();
-        }
-        catch (Exception e) {
-            throw new SQLException("Failed to query Ignite.", e);
+                rs = new JdbcResultSet(uuid, this, res.getTbls(), res.getCols(), res.getTypes(),
+                    res.getRows(), res.isFinished());
+
+                rs.setFetchSize(fetchSize);
+            }
+            catch (IgniteSQLException e) {
+                throw e.toJdbcException();
+            }
+            catch (Exception e) {
+                throw new SQLException("Failed to query Ignite.", e);
+            }
         }
+
+        resSets.add(rs);
+
+        return rs;
     }
 
     /** {@inheritDoc} */
@@ -165,8 +190,15 @@ long doUpdate(String sql, Object[] args) throws SQLException {
         if (!conn.isDmlSupported())
             throw new SQLException("Failed to query Ignite: DML operations are supported in versions 1.8.0 and newer");
 
-        JdbcQueryTaskV2 qryTask = new JdbcQueryTaskV2(loc ? ignite : null, conn.cacheName(), sql, false, loc, args,
-            fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(), conn.isDistributedJoins());
+        JdbcQueryTaskV2 qryTask;
+        if (conn.isEnforceJoinOrder()) {
+            qryTask = new JdbcQueryTaskV3(loc ? ignite : null, conn.cacheName(), sql, false, loc, args,
+                fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(), conn.isDistributedJoins(), true);
+        }
+        else {
+            qryTask = new JdbcQueryTaskV2(loc ? ignite : null, conn.cacheName(), sql, false, loc, args,
+                fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(), conn.isDistributedJoins());
+        }
 
         try {
             JdbcQueryTaskV2.QueryResult qryRes =
@@ -219,6 +251,7 @@ private static long updateCounterFromQueryResult(List<List<?>> rows) throws SQLE
 
     /**
      * Marks statement as closed and closes all result sets.
+     * @throws SQLException On error.
      */
     void closeInternal() throws SQLException {
         for (Iterator<JdbcResultSet> it = resSets.iterator(); it.hasNext(); ) {
@@ -332,8 +365,17 @@ void closeInternal() throws SQLException {
 
         boolean loc = nodeId == null;
 
-        JdbcQueryTaskV2 qryTask = new JdbcQueryTaskV2(loc ? ignite : null, conn.cacheName(), sql, null, loc, getArgs(),
-            fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(), conn.isDistributedJoins());
+        JdbcQueryTaskV2 qryTask;
+        if (conn.isEnforceJoinOrder()) {
+            qryTask = new JdbcQueryTaskV3(loc ? ignite : null, conn.cacheName(), sql, null, loc, getArgs(),
+                fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(),
+                conn.isDistributedJoins(), true);
+        }
+        else {
+            qryTask = new JdbcQueryTaskV2(loc ? ignite : null, conn.cacheName(), sql, null, loc, getArgs(),
+                fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(),
+                conn.isDistributedJoins());
+        }
 
         try {
             JdbcQueryTaskV2.QueryResult res =

From 5356817709fcc46a7ff8fd5e0abec184139b19b9 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 29 Jun 2017 17:11:39 +0300
Subject: [PATCH 431/516] Backport IGNITE-5613 into 1.8.x

IGNITE-5613 - Fixed deadlock on sequence update inside transaction

(cherry picked from commit 7db925c)
(cherry picked from commit 1488391f1e7b2687d212804da5a509896d452c87)
---
 .../GridCacheAtomicSequenceImpl.java          | 249 +++++-------------
 1 file changed, 64 insertions(+), 185 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
index 754d8f50a1ca9..8d636b44fe47b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
@@ -24,19 +24,16 @@
 import java.io.ObjectOutput;
 import java.io.ObjectStreamException;
 import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
-import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -45,8 +42,7 @@
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.Nullable;
 
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.apache.ignite.internal.util.typedef.internal.CU.retryTopologySafe;
+import static org.apache.ignite.internal.processors.cache.GridCacheUtils.retryTopologySafe;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
 import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
@@ -88,7 +84,7 @@ public final class GridCacheAtomicSequenceImpl implements GridCacheAtomicSequenc
 
     /** Local value of sequence. */
     @GridToStringInclude(sensitive = true)
-    private long locVal;
+    private volatile long locVal;
 
     /**  Upper bound of local counter. */
     private long upBound;
@@ -96,11 +92,14 @@ public final class GridCacheAtomicSequenceImpl implements GridCacheAtomicSequenc
     /**  Sequence batch size */
     private volatile int batchSize;
 
-    /** Synchronization lock. */
-    private final Lock lock = new ReentrantLock();
+    /** Synchronization lock for local value updates. */
+    private final Lock localUpdate = new ReentrantLock();
 
-    /** Await condition. */
-    private Condition cond = lock.newCondition();
+    /** Synchronization for distributed sequence update. Acquired by threads with free topology (not in TX). */
+    private final ReentrantLock distUpdateFreeTop = new ReentrantLock();
+
+    /** Synchronization for distributed sequence update. Acquired by threads with locked topology (inside TX). */
+    private final ReentrantLock distUpdateLockedTop = new ReentrantLock();
 
     /** Callable for execution {@link #incrementAndGet} operation in async and sync mode.  */
     private final Callable<Long> incAndGetCall = internalUpdate(1, true);
@@ -108,9 +107,6 @@ public final class GridCacheAtomicSequenceImpl implements GridCacheAtomicSequenc
     /** Callable for execution {@link #getAndIncrement} operation in async and sync mode.  */
     private final Callable<Long> getAndIncCall = internalUpdate(1, false);
 
-    /** Add and get cache call guard. */
-    private final AtomicBoolean updateGuard = new AtomicBoolean();
-
     /**
      * Empty constructor required by {@link Externalizable}.
      */
@@ -162,14 +158,7 @@ public GridCacheAtomicSequenceImpl(String name,
     @Override public long get() {
         checkRemoved();
 
-        lock.lock();
-
-        try {
-            return locVal;
-        }
-        finally {
-            lock.unlock();
-        }
+        return locVal;
     }
 
     /** {@inheritDoc} */
@@ -232,154 +221,51 @@ private long internalUpdate(long l, @Nullable Callable<Long> updateCall, boolean
 
         assert l > 0;
 
-        lock.lock();
+        localUpdate.lock();
 
         try {
             // If reserved range isn't exhausted.
-            if (locVal + l <= upBound) {
-                long curVal = locVal;
+            long locVal0 = locVal;
 
-                locVal += l;
+            if (locVal0 + l <= upBound) {
+                locVal = locVal0 + l;
 
-                return updated ? locVal : curVal;
+                return updated ? locVal0 + l : locVal0;
             }
         }
         finally {
-            lock.unlock();
+            localUpdate.unlock();
         }
 
-        if (updateCall == null)
-            updateCall = internalUpdate(l, updated);
-
-        while (true) {
-            if (updateGuard.compareAndSet(false, true)) {
-                try {
-                    try {
-                        return updateCall.call();
-                    }
-                    catch (IgniteCheckedException | IgniteException | IllegalStateException e) {
-                        throw e;
-                    }
-                    catch (Exception e) {
-                        throw new IgniteCheckedException(e);
-                    }
-                }
-                finally {
-                    lock.lock();
-
-                    try {
-                        updateGuard.set(false);
-
-                        cond.signalAll();
-                    }
-                    finally {
-                        lock.unlock();
-                    }
-                }
-            }
-            else {
-                lock.lock();
-
-                try {
-                    while (locVal >= upBound && updateGuard.get())
-                        U.await(cond, 500, MILLISECONDS);
+        AffinityTopologyVersion lockedVer = ctx.shared().lockedTopologyVersion(null);
 
-                    checkRemoved();
-
-                    // If reserved range isn't exhausted.
-                    if (locVal + l <= upBound) {
-                        long curVal = locVal;
-
-                        locVal += l;
+        // We need two separate locks here because two independent thread may attempt to update the sequence
+        // simultaneously, one thread with locked topology and other with unlocked.
+        // We cannot use the same lock for both cases because it leads to a deadlock when free-topology thread
+        // waits for topology change, and locked topology thread waits to acquire the lock.
+        // If a thread has locked topology, it must bypass sync with non-locked threads, but at the same time
+        // we do not want multiple threads to attempt to run identical cache updates.
+        ReentrantLock distLock = lockedVer == null ? distUpdateFreeTop : distUpdateLockedTop;
 
-                        return updated ? locVal : curVal;
-                    }
-                }
-                finally {
-                    lock.unlock();
-                }
-            }
-        }
-    }
-
-    /**
-     * Asynchronous sequence update operation. Will add given amount to the sequence value.
-     *
-     * @param l Increment amount.
-     * @param updateCall Cache call that will update sequence reservation count in accordance with l.
-     * @param updated If {@code true}, will return sequence value after update, otherwise will return sequence value
-     *      prior to update.
-     * @return Future indicating sequence value.
-     * @throws IgniteCheckedException If update failed.
-     */
-    @SuppressWarnings("SignalWithoutCorrespondingAwait")
-    private IgniteInternalFuture<Long> internalUpdateAsync(long l, @Nullable Callable<Long> updateCall, boolean updated)
-        throws IgniteCheckedException {
-        checkRemoved();
-
-        A.ensure(l > 0, " Parameter mustn't be less then 1: " + l);
-
-        lock.lock();
+        distLock.lock();
 
         try {
-            // If reserved range isn't exhausted.
-            if (locVal + l <= upBound) {
-                long curVal = locVal;
+            if (updateCall == null)
+                updateCall = internalUpdate(l, updated);
 
-                locVal += l;
-
-                return new GridFinishedFuture<>(updated ? locVal : curVal);
+            try {
+                return updateCall.call();
             }
-        }
-        finally {
-            lock.unlock();
-        }
-
-        if (updateCall == null)
-            updateCall = internalUpdate(l, updated);
-
-        while (true) {
-            if (updateGuard.compareAndSet(false, true)) {
-                try {
-                    // This call must be outside lock.
-                    return ctx.closures().callLocalSafe(updateCall, true);
-                }
-                finally {
-                    lock.lock();
-
-                    try {
-                        updateGuard.set(false);
-
-                        cond.signalAll();
-                    }
-                    finally {
-                        lock.unlock();
-                    }
-                }
+            catch (IgniteCheckedException | IgniteException | IllegalStateException e) {
+                throw e;
             }
-            else {
-                lock.lock();
-
-                try {
-                    while (locVal >= upBound && updateGuard.get())
-                        U.await(cond, 500, MILLISECONDS);
-
-                    checkRemoved();
-
-                    // If reserved range isn't exhausted.
-                    if (locVal + l <= upBound) {
-                        long curVal = locVal;
-
-                        locVal += l;
-
-                        return new GridFinishedFuture<>(updated ? locVal : curVal);
-                    }
-                }
-                finally {
-                    lock.unlock();
-                }
+            catch (Exception e) {
+                throw new IgniteCheckedException(e);
             }
         }
+        finally {
+            distLock.unlock();
+        }
     }
 
     /** Get local batch size for this sequences.
@@ -398,13 +284,13 @@ private IgniteInternalFuture<Long> internalUpdateAsync(long l, @Nullable Callabl
     @Override public void batchSize(int size) {
         A.ensure(size > 0, " Batch size can't be less then 0: " + size);
 
-        lock.lock();
+        localUpdate.lock();
 
         try {
             batchSize = size;
         }
         finally {
-            lock.unlock();
+            localUpdate.unlock();
         }
     }
 
@@ -486,6 +372,8 @@ private IllegalStateException removedError() {
     private Callable<Long> internalUpdate(final long l, final boolean updated) {
         return retryTopologySafe(new Callable<Long>() {
             @Override public Long call() throws Exception {
+                assert distUpdateFreeTop.isHeldByCurrentThread() || distUpdateLockedTop.isHeldByCurrentThread();
+
                 try (IgniteInternalTx tx = CU.txStartInternal(ctx, seqView, PESSIMISTIC, REPEATABLE_READ)) {
                     GridCacheAtomicSequenceValue seq = seqView.get(key);
 
@@ -497,48 +385,39 @@ private Callable<Long> internalUpdate(final long l, final boolean updated) {
 
                     long newUpBound;
 
-                    lock.lock();
-
-                    try {
-                        curLocVal = locVal;
+                    curLocVal = locVal;
 
-                        // If local range was already reserved in another thread.
-                        if (locVal + l <= upBound) {
-                            long retVal = locVal;
+                    // If local range was already reserved in another thread.
+                    if (curLocVal + l <= upBound) {
+                        locVal = curLocVal + l;
 
-                            locVal += l;
-
-                            return updated ? locVal : retVal;
-                        }
+                        return updated ? curLocVal + l : curLocVal;
+                    }
 
-                        long curGlobalVal = seq.get();
+                    long curGlobalVal = seq.get();
 
-                        long newLocVal;
+                    long newLocVal;
 
-                        /* We should use offset because we already reserved left side of range.*/
-                        long off = batchSize > 1 ? batchSize - 1 : 1;
+                    /* We should use offset because we already reserved left side of range.*/
+                    long off = batchSize > 1 ? batchSize - 1 : 1;
 
-                        // Calculate new values for local counter, global counter and upper bound.
-                        if (curLocVal + l >= curGlobalVal) {
-                            newLocVal = curLocVal + l;
+                    // Calculate new values for local counter, global counter and upper bound.
+                    if (curLocVal + l >= curGlobalVal) {
+                        newLocVal = curLocVal + l;
 
-                            newUpBound = newLocVal + off;
-                        }
-                        else {
-                            newLocVal = curGlobalVal;
+                        newUpBound = newLocVal + off;
+                    }
+                    else {
+                        newLocVal = curGlobalVal;
 
-                            newUpBound = newLocVal + off;
-                        }
+                        newUpBound = newLocVal + off;
+                    }
 
-                        locVal = newLocVal;
-                        upBound = newUpBound;
+                    locVal = newLocVal;
+                    upBound = newUpBound;
 
-                        if (updated)
-                            curLocVal = newLocVal;
-                    }
-                    finally {
-                        lock.unlock();
-                    }
+                    if (updated)
+                        curLocVal = newLocVal;
 
                     // Global counter must be more than reserved upper bound.
                     seq.set(newUpBound + 1);

From b38128dcea7e665ca81ce1dc8566f8b8dcda2e7e Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Mon, 3 Jul 2017 17:05:48 +0300
Subject: [PATCH 432/516] Backport IGNITE-5613 into 1.8.x

IGNITE-5613 - Fixed race on local sequence increment and distributed update

(cherry picked from commit 7d42dea)
(cherry picked from commit 9f977ec1602e17c01207e464f81cd747539ab6dc)
---
 .../GridCacheAtomicSequenceImpl.java          | 55 +++++++++++--------
 ...tionedAtomicSequenceMultiThreadedTest.java | 32 +++++++++++
 2 files changed, 64 insertions(+), 23 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
index 8d636b44fe47b..95e2f2fa44113 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java
@@ -385,39 +385,48 @@ private Callable<Long> internalUpdate(final long l, final boolean updated) {
 
                     long newUpBound;
 
-                    curLocVal = locVal;
+                    // Even though we hold a transaction lock here, we must hold the local update lock here as well
+                    // because we mutate multipe variables (locVal and upBound).
+                    localUpdate.lock();
 
-                    // If local range was already reserved in another thread.
-                    if (curLocVal + l <= upBound) {
-                        locVal = curLocVal + l;
+                    try {
+                        curLocVal = locVal;
 
-                        return updated ? curLocVal + l : curLocVal;
-                    }
+                        // If local range was already reserved in another thread.
+                        if (curLocVal + l <= upBound) {
+                            locVal = curLocVal + l;
 
-                    long curGlobalVal = seq.get();
+                            return updated ? curLocVal + l : curLocVal;
+                        }
 
-                    long newLocVal;
+                        long curGlobalVal = seq.get();
 
-                    /* We should use offset because we already reserved left side of range.*/
-                    long off = batchSize > 1 ? batchSize - 1 : 1;
+                        long newLocVal;
 
-                    // Calculate new values for local counter, global counter and upper bound.
-                    if (curLocVal + l >= curGlobalVal) {
-                        newLocVal = curLocVal + l;
+                        /* We should use offset because we already reserved left side of range.*/
+                        long off = batchSize > 1 ? batchSize - 1 : 1;
 
-                        newUpBound = newLocVal + off;
-                    }
-                    else {
-                        newLocVal = curGlobalVal;
+                        // Calculate new values for local counter, global counter and upper bound.
+                        if (curLocVal + l >= curGlobalVal) {
+                            newLocVal = curLocVal + l;
 
-                        newUpBound = newLocVal + off;
-                    }
+                            newUpBound = newLocVal + off;
+                        }
+                        else {
+                            newLocVal = curGlobalVal;
 
-                    locVal = newLocVal;
-                    upBound = newUpBound;
+                            newUpBound = newLocVal + off;
+                        }
 
-                    if (updated)
-                        curLocVal = newLocVal;
+                        locVal = newLocVal;
+                        upBound = newUpBound;
+
+                        if (updated)
+                            curLocVal = newLocVal;
+                    }
+                    finally {
+                        localUpdate.unlock();
+                    }
 
                     // Global counter must be more than reserved upper bound.
                     seq.set(newUpBound + 1);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSequenceMultiThreadedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSequenceMultiThreadedTest.java
index 945650da6b8f6..4db9bd3339904 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSequenceMultiThreadedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSequenceMultiThreadedTest.java
@@ -19,6 +19,7 @@
 
 import java.util.Random;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 import org.apache.ignite.IgniteAtomicSequence;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheMode;
@@ -26,6 +27,7 @@
 import org.apache.ignite.internal.processors.cache.datastructures.IgniteAtomicsAbstractTest;
 import org.apache.ignite.internal.processors.datastructures.GridCacheAtomicSequenceImpl;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
 
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 
@@ -280,6 +282,36 @@ public void testMixed2() throws Exception {
         assertEquals(17 * ITERATION_NUM * THREAD_NUM, seq.get());
     }
 
+    /**
+     * @throws Exception if failed.
+     */
+    public void testMultipleSequences() throws Exception {
+        final int seqCnt = 5;
+        final int threadCnt = 5;
+        final int incCnt = 1_000;
+
+        final IgniteAtomicSequence[] seqs = new IgniteAtomicSequence[seqCnt];
+
+        String seqName = UUID.randomUUID().toString();
+
+        for (int i = 0; i < seqs.length; i++)
+            seqs[i] = grid(0).atomicSequence(seqName, 0, true);
+
+        GridTestUtils.runMultiThreaded(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                for (int i = 0; i < incCnt; i++) {
+                    for (IgniteAtomicSequence seq : seqs)
+                        seq.incrementAndGet();
+                }
+
+                return null;
+            }
+        }, threadCnt, "load");
+
+        for (IgniteAtomicSequence seq : seqs)
+            assertEquals(seqCnt * threadCnt * incCnt, seq.get());
+    }
+
     /**
      * Executes given closure in a given number of threads given number of times.
      *

From 30b364adda12ee9a7344edf8a2dddf4083b2c037 Mon Sep 17 00:00:00 2001
From: nikolay_tikhonov <ntikhonov@gridgain.com>
Date: Wed, 15 Nov 2017 17:24:45 +0300
Subject: [PATCH 433/516] Fixed javadoc build.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>
---
 .../internal/processors/cache/query/GridCacheQueryRequest.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
index 4a7e14f9f8789..e00db0a4eb85f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
@@ -483,7 +483,7 @@ public int taskHash() {
     }
 
     /**
-     * This method needed for scan query. {@link #GridCacheQueryRequest#partition()} method related with IO policy and cannot be used
+     * This method needed for scan query. {@link #partition()} method related with IO policy and cannot be used
      * for this role. Otherwise Query will be processed in striped pool.
      *
      * @return partition.

From 5043121c5ae75f899d86368725ba5bc513e304b4 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 15 Nov 2017 16:04:50 +0300
Subject: [PATCH 434/516] GG-13060 - Fix NPE on started node on
 GridDhtPartitionsSingleRequest.

---
 .../GridCachePartitionExchangeManager.java    |  8 +++++++-
 .../GridDhtPartitionsExchangeFuture.java      | 19 +++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 93fe6a835053b..441201dfc4849 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -1357,7 +1357,13 @@ private void processSinglePartitionRequest(ClusterNode node, GridDhtPartitionsSi
             return;
 
         try {
-            sendLocalPartitions(node, msg.exchangeId());
+            final GridDhtPartitionsExchangeFuture exchFut = exchangeFuture(msg.exchangeId(),
+                null,
+                null,
+                null,
+                null);
+
+            exchFut.onReceivePartitionRequest(node);
         }
         finally {
             leaveBusy();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 59084f542566c..70311bd2914f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1615,6 +1615,25 @@ private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
         }
     }
 
+    /**
+     * @param node Sender node.
+     */
+    public void onReceivePartitionRequest(final ClusterNode node) {
+        assert !cctx.kernalContext().clientNode();
+        assert !node.isDaemon() && !CU.clientNode(node) : node;
+
+        initFut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
+            @Override public void apply(IgniteInternalFuture<Boolean> fut) {
+                try {
+                    sendLocalPartitions(node);
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to send message to coordinator: " + e);
+                }
+            }
+        });
+    }
+
     /**
      * @param node Sender node.
      * @param msg Full partition info.

From 039b44041741b862f584f160419d9612a7ee7366 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 16 Nov 2017 11:31:56 +0300
Subject: [PATCH 435/516] GG-13067 - Fix GridClockSyncProcessor may produce NPE
 on node stop.

---
 .../internal/processors/clock/GridClockSyncProcessor.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
index 3586956da99dd..c5cbbec4d6b24 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/clock/GridClockSyncProcessor.java
@@ -153,6 +153,9 @@ public GridClockSyncProcessor(GridKernalContext ctx) {
         try {
             stopping = false;
 
+            if (srv != null)
+                srv.beforeStop();
+
             if (timeCoord != null) {
                 timeCoord.cancel();
 
@@ -160,9 +163,6 @@ public GridClockSyncProcessor(GridKernalContext ctx) {
 
                 timeCoord = null;
             }
-
-            if (srv != null)
-                srv.beforeStop();
         }
         finally {
             rw.writeUnlock();

From 5bfb748d54155f3a008c9f760487d8f967798bbc Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Thu, 16 Nov 2017 15:12:20 +0300
Subject: [PATCH 436/516] GG-12706 Unmute IgniteCacheMessageWriteTimeoutTest
 fixed by IGNITE-6818.

---
 .../cache/distributed/IgniteCacheMessageWriteTimeoutTest.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index 8f3f7723efb60..da9f11f6a7dd8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -75,8 +75,6 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testMessageQueueLimit() throws Exception {
-        fail("https://ggsystems.atlassian.net/browse/GG-12398");
-
         for (int i = 0; i < 15; i++) {
             log.info("Iteration: " + i);
 

From 0d6adf7c0dc1b40177e9cf630036e6b7979f4024 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Thu, 25 May 2017 20:19:22 +0300
Subject: [PATCH 437/516] Backported ignite-gg-12138 stop nodes after test +
 small test refactoring

(cherry picked from commit 54beab9)
---
 .../tcp/ipfinder/TcpDiscoveryIpFinderAbstractSelfTest.java | 7 +++++++
 .../tcp/ipfinder/vm/TcpDiscoveryVmIpFinderSelfTest.java    | 7 +++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/TcpDiscoveryIpFinderAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/TcpDiscoveryIpFinderAbstractSelfTest.java
index 6f2201f8d34e4..465b38dbe7859 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/TcpDiscoveryIpFinderAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/TcpDiscoveryIpFinderAbstractSelfTest.java
@@ -54,6 +54,13 @@ protected TcpDiscoveryIpFinderAbstractSelfTest() throws Exception {
         injectLogger(finder);
     }
 
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
     /**
      * @throws Exception If any error occurs.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/vm/TcpDiscoveryVmIpFinderSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/vm/TcpDiscoveryVmIpFinderSelfTest.java
index aa000071647d3..85f1a9ddf29d0 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/vm/TcpDiscoveryVmIpFinderSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/vm/TcpDiscoveryVmIpFinderSelfTest.java
@@ -223,6 +223,7 @@ public void testUnregistration() throws Exception {
 
         Ignition.stop("client1", true);
         Ignition.stop("client2", true);
+        Ignition.stop("client3", true);
 
         assertEquals(3 * srvSize, IP_FINDER.getRegisteredAddresses().size());
 
@@ -237,12 +238,14 @@ public void testUnregistration() throws Exception {
         Ignition.stop("server1", true);
         Ignition.stop("server2", true);
 
-        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+        boolean res = GridTestUtils.waitForCondition(new GridAbsPredicate() {
             @Override public boolean apply() {
-                return 0 == G.allGrids().size();
+                return G.allGrids().isEmpty();
             }
         }, 10000);
 
+        assertTrue(res);
+
         assertTrue(3 * srvSize >= IP_FINDER.getRegisteredAddresses().size());
     }
 

From 2af9c2615e7156e246bb0d662f14dbc9c4e13490 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Fri, 9 Jun 2017 00:34:37 +0300
Subject: [PATCH 438/516] Fixed missed node stop after tests.

(cherry picked from commit 2252510)
---
 .../internal/ClusterGroupHostsSelfTest.java   |  7 ++
 .../internal/GridStartStopSelfTest.java       | 65 ++++++++++---------
 ...dCachePartitionedUnloadEventsSelfTest.java | 48 ++++++++------
 .../near/GridCacheNearTxForceKeyTest.java     | 21 +++---
 .../near/NearCacheSyncUpdateTest.java         |  7 ++
 5 files changed, 90 insertions(+), 58 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java
index d0251928e3f36..9b0881a6788c9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java
@@ -44,6 +44,13 @@ public class ClusterGroupHostsSelfTest extends GridCommonAbstractTest {
         startGrid();
     }
 
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         Collection<String> hostNames = Arrays.asList("h_1", "h_2", "h_3");
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridStartStopSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridStartStopSelfTest.java
index a08db921bf76e..c0540d3a89508 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridStartStopSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridStartStopSelfTest.java
@@ -85,54 +85,61 @@ public void testStopWhileInUse() throws Exception {
 
         cfg.setCacheConfiguration(cc);
 
-        final Ignite g0 = G.start(cfg);
+        try {
+            final Ignite g0 = G.start(cfg);
 
-        cfg = new IgniteConfiguration();
+            cfg = new IgniteConfiguration();
 
-        cfg.setGridName(getTestGridName(1));
+            cfg.setGridName(getTestGridName(1));
 
-        cc = new CacheConfiguration();
+            cc = new CacheConfiguration();
 
-        cc.setAtomicityMode(TRANSACTIONAL);
+            cc.setAtomicityMode(TRANSACTIONAL);
 
-        cfg.setCacheConfiguration(cc);
+            cfg.setCacheConfiguration(cc);
 
-        final CountDownLatch latch = new CountDownLatch(1);
+            final CountDownLatch latch = new CountDownLatch(1);
 
-        Ignite g1 = G.start(cfg);
+            Ignite g1 = G.start(cfg);
 
-        Thread stopper = new Thread(new Runnable() {
-            @Override public void run() {
-                try {
-                    try (Transaction ignored = g0.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
-                        g0.cache(null).get(1);
+            Thread stopper = new Thread(new Runnable() {
+                @Override public void run() {
+                    try {
+                        try (Transaction ignored = g0.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                            g0.cache(null).get(1);
 
-                        latch.countDown();
+                            latch.countDown();
 
-                        Thread.sleep(500);
+                            Thread.sleep(500);
 
-                        info("Before stop.");
+                            info("Before stop.");
 
-                        G.stop(getTestGridName(1), true);
+                            G.stop(getTestGridName(1), true);
+                        }
+                    }
+                    catch (Exception e) {
+                        error("Error.", e);
                     }
                 }
-                catch (Exception e) {
-                    error("Error.", e);
-                }
-            }
-        });
+            });
 
-        stopper.start();
+            stopper.start();
 
-        assert latch.await(1, SECONDS);
+            assert latch.await(1, SECONDS);
 
-        info("Before remove.");
+            info("Before remove.");
 
-        try {
-            g1.cache(null).remove(1);
+            try {
+                g1.cache(null).remove(1);
+            }
+            catch (CacheException ignore) {
+                // No-op.
+            }
+
+            stopper.join();
         }
-        catch (CacheException ignore) {
-            // No-op.
+        finally {
+            stopAllGrids();
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedUnloadEventsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedUnloadEventsSelfTest.java
index 98f3cd71f9f78..9912bfba06c3c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedUnloadEventsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedUnloadEventsSelfTest.java
@@ -47,6 +47,9 @@ public class GridCachePartitionedUnloadEventsSelfTest extends GridCommonAbstract
     /** */
     private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
 
+    /** */
+    private static final int EVENTS_COUNT = 40;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -76,38 +79,41 @@ protected CacheConfiguration cacheConfiguration() {
      * @throws Exception if failed.
      */
     public void testUnloadEvents() throws Exception {
-        final Ignite g1 = startGrid("g1");
-
-        Collection<Integer> allKeys = new ArrayList<>(100);
+        try {
+            final Ignite g1 = startGrid("g1");
 
-        IgniteCache<Integer, String> cache = g1.cache(null);
+            Collection<Integer> allKeys = new ArrayList<>(EVENTS_COUNT);
 
-        for (int i = 0; i < 100; i++) {
-            cache.put(i, "val");
-            allKeys.add(i);
-        }
+            IgniteCache<Integer, String> cache = g1.cache(null);
 
-        Ignite g2 = startGrid("g2");
+            for (int i = 0; i < EVENTS_COUNT; i++) {
+                cache.put(i, "val");
+                allKeys.add(i);
+            }
 
-        awaitPartitionMapExchange();
+            Ignite g2 = startGrid("g2");
 
-        Map<ClusterNode, Collection<Object>> keysMap = g1.affinity(null).mapKeysToNodes(allKeys);
-        Collection<Object> g2Keys = keysMap.get(g2.cluster().localNode());
+            awaitPartitionMapExchange();
 
-        assertNotNull(g2Keys);
-        assertFalse("There are no keys assigned to g2", g2Keys.isEmpty());
+            Map<ClusterNode, Collection<Object>> keysMap = g1.affinity(null).mapKeysToNodes(allKeys);
+            Collection<Object> g2Keys = keysMap.get(g2.cluster().localNode());
 
-        Thread.sleep(5000);
+            assertNotNull(g2Keys);
+            assertFalse("There are no keys assigned to g2", g2Keys.isEmpty());
 
-        Collection<Event> objEvts =
-            g1.events().localQuery(F.<Event>alwaysTrue(), EVT_CACHE_REBALANCE_OBJECT_UNLOADED);
+            Collection<Event> objEvts =
+                g1.events().localQuery(F.<Event>alwaysTrue(), EVT_CACHE_REBALANCE_OBJECT_UNLOADED);
 
-        checkObjectUnloadEvents(objEvts, g1, g2Keys);
+            checkObjectUnloadEvents(objEvts, g1, g2Keys);
 
-        Collection <Event> partEvts =
-            g1.events().localQuery(F.<Event>alwaysTrue(), EVT_CACHE_REBALANCE_PART_UNLOADED);
+            Collection<Event> partEvts =
+                g1.events().localQuery(F.<Event>alwaysTrue(), EVT_CACHE_REBALANCE_PART_UNLOADED);
 
-        checkPartitionUnloadEvents(partEvts, g1, dht(g2.cache(null)).topology().localPartitions());
+            checkPartitionUnloadEvents(partEvts, g1, dht(g2.cache(null)).topology().localPartitions());
+        }
+        finally {
+            stopAllGrids();
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxForceKeyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxForceKeyTest.java
index 80b447cab86be..f1b17262dcccb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxForceKeyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxForceKeyTest.java
@@ -65,19 +65,24 @@ public class GridCacheNearTxForceKeyTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testNearTx() throws Exception {
-        Ignite ignite0 = startGrid(0);
+        try {
+            Ignite ignite0 = startGrid(0);
 
-        IgniteCache<Integer, Integer> cache = ignite0.cache(null);
+            IgniteCache<Integer, Integer> cache = ignite0.cache(null);
 
-        Ignite ignite1 = startGrid(1);
+            Ignite ignite1 = startGrid(1);
 
-        // This key should become primary for ignite1.
-        final Integer key = ignite0.configuration().getMarshaller() instanceof OptimizedMarshaller ? 2 : 7;
+            // This key should become primary for ignite1.
+            final Integer key = ignite0.configuration().getMarshaller() instanceof OptimizedMarshaller ? 2 : 7;
 
-        assertNull(cache.getAndPut(key, key));
+            assertNull(cache.getAndPut(key, key));
 
-        awaitPartitionMapExchange();
+            awaitPartitionMapExchange();
 
-        assertTrue(ignite0.affinity(null).isPrimary(ignite1.cluster().localNode(), key));
+            assertTrue(ignite0.affinity(null).isPrimary(ignite1.cluster().localNode(), key));
+        }
+        finally {
+            stopAllGrids();
+        }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NearCacheSyncUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NearCacheSyncUpdateTest.java
index bbdf50ec7e45f..b29bbbc6ab8f5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NearCacheSyncUpdateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NearCacheSyncUpdateTest.java
@@ -63,6 +63,13 @@ public class NearCacheSyncUpdateTest extends GridCommonAbstractTest {
         startGridsMultiThreaded(3);
     }
 
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
     /**
      * @throws Exception If failed.
      */

From 67e2bdf04c84145ea6113cf441042a3dfdc9957b Mon Sep 17 00:00:00 2001
From: Vitaliy Biryukov <biryukovvitaliy92@gmail.com>
Date: Tue, 19 Sep 2017 13:23:45 +0300
Subject: [PATCH 439/516] Fixed missed node stop after tests.

---
 .../cache/query/IgniteCacheQueryCacheDestroySelfTest.java  | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IgniteCacheQueryCacheDestroySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IgniteCacheQueryCacheDestroySelfTest.java
index dc104ff229fe5..be52fc262e257 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IgniteCacheQueryCacheDestroySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IgniteCacheQueryCacheDestroySelfTest.java
@@ -49,6 +49,13 @@ public class IgniteCacheQueryCacheDestroySelfTest extends GridCommonAbstractTest
     /** */
     public static final int GRID_CNT = 3;
 
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
     /**
      * The main test code.
      */

From 7e6ba3310dddfd45c50d3aff44ebf8bdd7edeb7c Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 17 Nov 2017 16:43:35 +0300
Subject: [PATCH 440/516] ignite-6924: Fixed missed
 CacheStoreSessionListener#onSessionStart() call

---
 .../cache/store/CacheStoreManager.java        |  12 +-
 .../store/GridCacheStoreManagerAdapter.java   |   7 +
 .../store/GridCacheWriteBehindStore.java      |  10 +-
 ...SessionListenerWriteBehindEnabledTest.java | 124 ++++++++++++++++--
 4 files changed, 140 insertions(+), 13 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
index b096edf6a8b95..9938a1cd87479 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
@@ -172,12 +172,22 @@ public boolean removeAll(@Nullable IgniteInternalTx tx, Collection<Object> keys)
     public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last, boolean storeSessionEnded) throws IgniteCheckedException;
 
     /**
-     * End session initiated by write-behind store.
+     * Start session initiated by write-behind store.
      *
      * @throws IgniteCheckedException If failed.
      */
     public void writeBehindSessionInit() throws IgniteCheckedException;
 
+    /**
+     * Notifies cache store session listeners.
+     *
+     * This method is called by write-behind store in case of back-pressure mechanism is initiated.
+     * It is assumed that cache store session was started by CacheStoreManager before.
+     *
+     * @throws IgniteCheckedException If failed.
+     */
+    public void writeBehindCacheStoreSessionListenerStart()  throws IgniteCheckedException;
+
     /**
      * End session initiated by write-behind store.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 3dbec4d21a56a..937b85630264b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -839,6 +839,13 @@ private void handleClassCastException(ClassCastException e) throws IgniteChecked
         sessionInit0(null, null, true);
     }
 
+    /** {@inheritDoc} */
+    @Override public void writeBehindCacheStoreSessionListenerStart()  throws IgniteCheckedException {
+        assert sesHolder.get() != null;
+
+        notifyCacheStoreSessionListeners(sesHolder.get(), null, true);
+    }
+
     /** {@inheritDoc} */
     @Override public void writeBehindSessionEnd(boolean threwEx) throws IgniteCheckedException {
         sessionEnd0(null, threwEx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
index 8537aaba69282..82f88af3ef1b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
@@ -795,8 +795,14 @@ private boolean updateStore(
         Flusher flusher
     ) {
         try {
-            if (initSes && storeMgr != null)
-                storeMgr.writeBehindSessionInit();
+            if (storeMgr != null) {
+                if (initSes)
+                    storeMgr.writeBehindSessionInit();
+                else
+                    // Back-pressure mechanism is running.
+                    // Cache store session must be initialized by storeMgr.
+                    storeMgr.writeBehindCacheStoreSessionListenerStart();
+            }
 
             boolean threwEx = true;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java
index 2b04309a40422..143a16b99a3f3 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java
@@ -26,6 +26,8 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Logger;
 import javax.cache.Cache;
@@ -34,14 +36,15 @@
 import javax.cache.integration.CacheLoaderException;
 import javax.cache.integration.CacheWriterException;
 import javax.sql.DataSource;
-import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cache.store.jdbc.CacheJdbcStoreSessionListener;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
 import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore;
-import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.CacheStoreSessionResource;
+import org.apache.ignite.testframework.GridTestUtils;
 
 /**
  * This class tests that calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)}
@@ -64,14 +67,17 @@ public class CacheStoreSessionListenerWriteBehindEnabledTest extends GridCacheAb
     /** */
     private static final AtomicInteger entryCnt = new AtomicInteger();
 
+    /** */
+    private static final AtomicInteger uninitializedListenerCnt = new AtomicInteger();
+
     /** {@inheritDoc} */
     @Override protected int gridCount() {
         return 1;
     }
 
     /** {@inheritDoc} */
-    @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
-        CacheConfiguration cacheCfg = super.cacheConfiguration(igniteInstanceName);
+    @Override protected CacheConfiguration cacheConfiguration(String gridName) throws Exception {
+        CacheConfiguration cacheCfg = super.cacheConfiguration(gridName);
 
         cacheCfg.setName(DEFAULT_CACHE_NAME);
 
@@ -98,6 +104,8 @@ public class CacheStoreSessionListenerWriteBehindEnabledTest extends GridCacheAb
         operations.clear();
 
         entryCnt.set(0);
+
+        uninitializedListenerCnt.set(0);
     }
 
     /**
@@ -140,6 +148,85 @@ public void testRemove() {
         checkSessionCounters(1);
     }
 
+    /**
+     * Tests that cache store session listeners are notified by write-behind store.
+     */
+    public void testFlushSingleValue() throws Exception {
+        CacheConfiguration cfg = cacheConfiguration(getTestGridName());
+
+        cfg.setName("back-pressure-control");
+        cfg.setWriteBehindBatchSize(2);
+        cfg.setWriteBehindFlushSize(2);
+        cfg.setWriteBehindFlushFrequency(1_000);
+        cfg.setWriteBehindCoalescing(true);
+
+        IgniteCache<Object, Object> cache = grid(0).getOrCreateCache(cfg);
+
+        try {
+            int nUploaders = 5;
+
+            final CyclicBarrier barrier = new CyclicBarrier(nUploaders);
+
+            IgniteInternalFuture[] uploaders = new IgniteInternalFuture[nUploaders];
+
+            for (int i = 0; i < nUploaders; ++i) {
+                uploaders[i] = GridTestUtils.runAsync(
+                    new Uploader(cache, barrier, i * CNT),
+                    "uploader-" + i);
+            }
+
+            for (int i = 0; i < nUploaders; ++i)
+                uploaders[i].get();
+
+            assertEquals("Uninitialized cache store session listener.", 0, uninitializedListenerCnt.get());
+        }
+        finally {
+            cache.destroy();
+        }
+    }
+
+    /**
+     *
+     */
+    public static class Uploader implements Callable<Void> {
+        /** */
+        private final int start;
+
+        /** */
+        private final CyclicBarrier barrier;
+
+        /** */
+        private final IgniteCache<Object, Object> cache;
+
+        /**
+         * @param cache Ignite cache.
+         * @param barrier Cyclic barrier.
+         * @param start Key index.
+         */
+        public Uploader(IgniteCache<Object, Object> cache, CyclicBarrier barrier, int start) {
+            this.cache = cache;
+
+            this.barrier = barrier;
+
+            this.start = start;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Void call() {
+            try {
+                barrier.await();
+
+                for (int i = start; i < start + CNT; ++i)
+                    cache.put(i, i);
+            }
+            catch (Exception e) {
+                fail("Unexpected exception [" + e + "]");
+            }
+
+            return null;
+        }
+    }
+
     /**
      * @param startedSessions Number of expected sessions.
      */
@@ -150,6 +237,8 @@ private void checkSessionCounters(int startedSessions) {
 
             assertEquals(CNT, entryCnt.get());
 
+            assertEquals("Uninitialized cache store session listener.", 0, uninitializedListenerCnt.get());
+
             checkOpCount(operations, OperationType.SESSION_START, startedSessions);
 
             checkOpCount(operations, OperationType.SESSION_END, startedSessions);
@@ -206,18 +295,19 @@ public static class CacheStoreSessionFactory implements Factory<TestCacheStoreSe
      * Test cache store session listener.
      */
     public static class TestCacheStoreSessionListener extends CacheJdbcStoreSessionListener {
-        /** */
-        @IgniteInstanceResource
-        private Ignite ignite;
-
         /** {@inheritDoc} */
         @Override public void onSessionStart(CacheStoreSession ses) {
             operations.add(OperationType.SESSION_START);
+
+            if (ses.attachment() == null)
+                ses.attach(new Object());
         }
 
         /** {@inheritDoc} */
         @Override public void onSessionEnd(CacheStoreSession ses, boolean commit) {
             operations.add(OperationType.SESSION_END);
+
+            ses.attach(null);
         }
     }
 
@@ -229,31 +319,45 @@ public static class TestCacheStoreSessionListener extends CacheJdbcStoreSessionL
      */
     public static class EmptyCacheStore extends CacheStoreAdapter<Object, Object> {
         /** */
-        @IgniteInstanceResource
-        private Ignite ignite;
+        @CacheStoreSessionResource
+        private CacheStoreSession ses;
 
         /** {@inheritDoc} */
         @Override public Object load(Object key) throws CacheLoaderException {
             entryCnt.getAndIncrement();
+
+            if (ses.attachment() == null)
+                uninitializedListenerCnt.incrementAndGet();
+
             return null;
         }
 
         /** {@inheritDoc} */
         @Override public void writeAll(Collection<Cache.Entry<?, ?>> entries) {
             entryCnt.addAndGet(entries.size());
+
+            if (ses.attachment() == null)
+                uninitializedListenerCnt.incrementAndGet();
         }
 
         /** {@inheritDoc} */
         @Override public void write(Cache.Entry entry) throws CacheWriterException {
+            if (ses.attachment() == null)
+                uninitializedListenerCnt.incrementAndGet();
         }
 
         /** {@inheritDoc} */
         @Override public void deleteAll(Collection<?> keys) {
             entryCnt.addAndGet(keys.size());
+
+            if (ses.attachment() == null)
+                uninitializedListenerCnt.incrementAndGet();
         }
 
         /** {@inheritDoc} */
         @Override public void delete(Object key) throws CacheWriterException {
+            if (ses.attachment() == null)
+                uninitializedListenerCnt.incrementAndGet();
         }
     }
 

From 53e93afd936e8a257c99dddfff5222c40464738a Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 17 Nov 2017 15:53:15 +0300
Subject: [PATCH 441/516] IGNITE-6496 Client node should release queue
 semaphore on disconnect - Fixes #2981.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../CacheDataStructuresManager.java           |  12 ++
 .../datastructures/GridCacheQueueAdapter.java |   5 +
 .../GridCacheQueueClientDisconnectTest.java   | 119 ++++++++++++++++++
 ...gniteCacheDataStructuresSelfTestSuite.java |   2 +
 4 files changed, 138 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index 2b3080981ec36..92e5d51e78e72 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -62,6 +62,7 @@
 import org.apache.ignite.internal.util.GridSpinBusyLock;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.jetbrains.annotations.NotNull;
@@ -135,6 +136,17 @@ public CacheDataStructuresManager() {
             q.delegate().onKernalStop();
     }
 
+    /** {@inheritDoc} */
+    @Override public void onDisconnected(IgniteFuture reconnectFut) {
+        super.onDisconnected(reconnectFut);
+
+        for (Map.Entry<IgniteUuid, GridCacheQueueProxy> e : queuesMap.entrySet()) {
+            GridCacheQueueProxy queue = e.getValue();
+
+            queue.delegate().onClientDisconnected();
+        }
+    }
+
     /**
      * @param set Set.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java
index 6e087e69c3017..41b6161a1a618 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java
@@ -486,6 +486,11 @@ protected final void checkRemoved(@Nullable GridCacheQueueHeader hdr) {
             onRemoved(true);
     }
 
+    /** Release all semaphores used in blocking operations in case of client disconnect. */
+    public void onClientDisconnected() {
+        releaseSemaphores();
+    }
+
     /**
      * Marks queue as removed.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java
new file mode 100644
index 0000000000000..98f3544bdeeb4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.datastructures;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteClientDisconnectedException;
+import org.apache.ignite.IgniteQueue;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CollectionConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+public class GridCacheQueueClientDisconnectTest extends GridCommonAbstractTest {
+    /** */
+    private static final String IGNITE_QUEUE_NAME = "ignite-queue-client-reconnect-test";
+
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int FAILURE_DETECTION_TIMEOUT = 10_000;
+
+    /** */
+    private boolean clientMode;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi spi = new TcpDiscoverySpi();
+
+        spi.setIpFinder(ipFinder);
+
+        spi.setClientReconnectDisabled(false);
+
+        cfg.setDiscoverySpi(spi);
+
+        cfg.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT);
+
+        if (clientMode)
+            cfg.setClientMode(true);
+
+        return cfg;
+    }
+
+    private static CollectionConfiguration collectionConfiguration(CacheAtomicityMode cacheAtomicityMode) {
+        CollectionConfiguration colCfg = new CollectionConfiguration();
+
+        colCfg.setAtomicityMode(cacheAtomicityMode);
+
+        return colCfg;
+    }
+
+    public void testClientDisconnect() throws Exception {
+        try {
+            Ignite server = startGrid(0);
+
+            clientMode = true;
+
+            Ignite client = startGrid(1);
+
+            awaitPartitionMapExchange();
+
+            final IgniteQueue queue = client.queue(
+                IGNITE_QUEUE_NAME, 0, collectionConfiguration(CacheAtomicityMode.ATOMIC));
+
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() {
+                    try {
+                        Object value = queue.take();
+                    }
+                    catch (IgniteClientDisconnectedException icd) {
+                        latch.countDown();
+                    }
+                    catch (Exception e) {
+                    }
+
+                    return null;
+                }
+            });
+
+            U.sleep(5000);
+
+            server.close();
+
+            boolean countReachedZero = latch.await(FAILURE_DETECTION_TIMEOUT * 2, TimeUnit.MILLISECONDS);
+
+            assertTrue("IgniteClientDisconnectedException was not thrown", countReachedZero);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
index 980d550ecd608..bb0e1a3f16413 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.datastructures.GridCacheQueueCleanupSelfTest;
+import org.apache.ignite.internal.processors.cache.datastructures.GridCacheQueueClientDisconnectTest;
 import org.apache.ignite.internal.processors.cache.datastructures.GridCacheQueueMultiNodeConsistencySelfTest;
 import org.apache.ignite.internal.processors.cache.datastructures.IgniteClientDataStructuresTest;
 import org.apache.ignite.internal.processors.cache.datastructures.IgniteClientDiscoveryDataStructuresTest;
@@ -133,6 +134,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(GridCachePartitionedQueueMultiNodeSelfTest.class));
         suite.addTest(new TestSuite(GridCachePartitionedAtomicQueueMultiNodeSelfTest.class));
         suite.addTest(new TestSuite(GridCachePartitionedAtomicOffheapQueueMultiNodeSelfTest.class));
+        suite.addTest(new TestSuite(GridCacheQueueClientDisconnectTest.class));
 
         suite.addTest(new TestSuite(GridCachePartitionedQueueCreateMultiNodeSelfTest.class));
         suite.addTest(new TestSuite(GridCachePartitionedAtomicQueueCreateMultiNodeSelfTest.class));

From 75eef0dbc53710fa9d572a60e71af308634b5f63 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 17 Nov 2017 12:54:26 +0300
Subject: [PATCH 442/516] GG-13072 - Fix half open connection when server
 initiated connection.

---
 .../spi/communication/tcp/TcpCommunicationSpi.java    | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 55e2b6b95af98..1432b51e8454b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -529,7 +529,7 @@ private void onFirstMessage(final GridNioSession ses, Message msg) {
                         if (c.failed) {
                             ses.send(new RecoveryLastReceivedMessage(-1));
 
-                            closeStaleConnections(connKey);
+                            closeStaleConnections(connKey, true);
                         }
                     }
                 }
@@ -554,7 +554,7 @@ private void onFirstMessage(final GridNioSession ses, Message msg) {
 
                             ses.send(new RecoveryLastReceivedMessage(-1));
 
-                            closeStaleConnections(connKey);
+                            closeStaleConnections(connKey, false);
 
                             return;
                         }
@@ -588,7 +588,7 @@ private void onFirstMessage(final GridNioSession ses, Message msg) {
 
                                 ses.send(new RecoveryLastReceivedMessage(-1));
 
-                                closeStaleConnections(connKey);
+                                closeStaleConnections(connKey, false);
 
                                 fut.onDone(oldClient);
 
@@ -645,12 +645,13 @@ private void onFirstMessage(final GridNioSession ses, Message msg) {
 
             /**
              * @param connKey Connection key.
+             * @param paired Paired connection.
              */
-            private void closeStaleConnections(ConnectionKey connKey) {
+            private void closeStaleConnections(ConnectionKey connKey, boolean paired) {
                 for (GridNioSession ses0 : nioSrvr.sessions()) {
                     ConnectionKey key0 = ses0.meta(CONN_IDX_META);
 
-                    if (ses0.accepted() && key0 != null &&
+                    if ((!paired || ses0.accepted()) && key0 != null &&
                         key0.nodeId().equals(connKey.nodeId()) &&
                         key0.connectionIndex() == connKey.connectionIndex() &&
                         key0.connectCount() < connKey.connectCount())

From db0c0a024caf972f7b16e3d35bfca028e9ef3fc7 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Fri, 17 Nov 2017 18:55:05 +0300
Subject: [PATCH 443/516] fixed JdbcStreamingSelfTest,
 JdbcAbstractDmlStatementSelfTest

---
 .../internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java      | 4 ++++
 .../apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java   | 1 +
 2 files changed, 5 insertions(+)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
index 4965acff26b7d..f37e60e898cf1 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
@@ -207,7 +207,11 @@ protected String getCfgUrl() {
 
         grid(0).cache(null).clear();
 
+        conn.close();
+
         assertEquals(0, grid(0).cache(null).size(CachePeekMode.ALL));
+
+        assertTrue(conn.isClosed());
     }
 
     /**
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index ead2a6e6d4563..8a0dc6153bcff 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -72,6 +72,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcInsertStatementSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcBinaryMarshallerInsertStatementSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcDeleteStatementSelfTest.class));
+//        suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcUpdateStatementSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStreamingSelfTest.class));
         suite.addTest(new TestSuite(JdbcBlobTest.class));
 

From 7b7ac5f5c90666932675f85ceb6b2bc81064c759 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Fri, 17 Nov 2017 17:35:43 +0300
Subject: [PATCH 444/516] GG-13064 increase number of partitions in
 GridCacheRebalancingPartitionDistributionTest

(cherry picked from commit 84df3f5494b35ee4f755d49f7cb3be21ba9e4595)
---
 .../GridCacheRebalancingPartitionDistributionTest.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionDistributionTest.java
index 61ee9ea9338c3..eebafed1262f8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionDistributionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionDistributionTest.java
@@ -52,7 +52,7 @@ public class GridCacheRebalancingPartitionDistributionTest extends GridRollingRe
                 .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
                 .setCacheMode(CacheMode.PARTITIONED)
                 .setBackups(1)
-                .setAffinity(new RendezvousAffinityFunction(true /* machine-safe */, 271))
+                .setAffinity(new RendezvousAffinityFunction(true /* machine-safe */, 1024))
                 .setAtomicWriteOrderMode(CacheAtomicWriteOrderMode.CLOCK)
                 .setRebalanceMode(CacheRebalanceMode.SYNC)
                 .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);

From a68ec9f52250e626b25533ae406a237b2b6ab54e Mon Sep 17 00:00:00 2001
From: Konstantin Dudkov <kdudkov@ya.ru>
Date: Mon, 20 Nov 2017 17:33:38 +0300
Subject: [PATCH 445/516] Sanity test suite added

---
 .../testsuites/IgniteSanityTestSuite.java     | 160 ++++++++++++++++++
 1 file changed, 160 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSanityTestSuite.java

diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSanityTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSanityTestSuite.java
new file mode 100644
index 0000000000000..7763fe5795987
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSanityTestSuite.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import java.util.Set;
+import junit.framework.TestSuite;
+import org.apache.ignite.GridSuppressedExceptionSelfTest;
+import org.apache.ignite.internal.ClusterGroupHostsSelfTest;
+import org.apache.ignite.internal.ClusterGroupSelfTest;
+import org.apache.ignite.internal.GridFailFastNodeFailureDetectionSelfTest;
+import org.apache.ignite.internal.GridLifecycleAwareSelfTest;
+import org.apache.ignite.internal.GridLifecycleBeanSelfTest;
+import org.apache.ignite.internal.GridNodeMetricsLogSelfTest;
+import org.apache.ignite.internal.GridProjectionForCachesSelfTest;
+import org.apache.ignite.internal.GridReduceSelfTest;
+import org.apache.ignite.internal.GridReleaseTypeSelfTest;
+import org.apache.ignite.internal.GridSelfTest;
+import org.apache.ignite.internal.GridStartStopSelfTest;
+import org.apache.ignite.internal.GridStopWithCancelSelfTest;
+import org.apache.ignite.internal.IgniteLocalNodeMapBeforeStartTest;
+import org.apache.ignite.internal.IgniteSlowClientDetectionSelfTest;
+import org.apache.ignite.internal.MarshallerContextLockingSelfTest;
+import org.apache.ignite.internal.processors.affinity.GridAffinityProcessorRendezvousSelfTest;
+import org.apache.ignite.internal.processors.cache.GridLocalIgniteSerializationTest;
+import org.apache.ignite.internal.processors.cache.GridProjectionForCachesOnDaemonNodeSelfTest;
+import org.apache.ignite.internal.processors.cache.IgniteDaemonNodeMarshallerCacheTest;
+import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheConcurrentReadWriteTest;
+import org.apache.ignite.internal.processors.cache.OffHeapTieredTransactionSelfTest;
+import org.apache.ignite.internal.processors.closure.GridClosureProcessorSelfTest;
+import org.apache.ignite.internal.processors.closure.GridClosureSerializationTest;
+import org.apache.ignite.internal.processors.continuous.GridEventConsumeSelfTest;
+import org.apache.ignite.internal.processors.continuous.GridMessageListenSelfTest;
+import org.apache.ignite.internal.processors.odbc.OdbcEscapeSequenceSelfTest;
+import org.apache.ignite.internal.processors.odbc.OdbcProcessorValidationSelfTest;
+import org.apache.ignite.internal.processors.service.ClosureServiceClientsNodesTest;
+import org.apache.ignite.internal.product.GridProductVersionSelfTest;
+import org.apache.ignite.internal.util.nio.IgniteExceptionInNioWorkerSelfTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.marshaller.DynamicProxySerializationMultiJvmSelfTest;
+import org.apache.ignite.marshaller.MarshallerContextSelfTest;
+import org.apache.ignite.messaging.GridMessagingNoPeerClassLoadingSelfTest;
+import org.apache.ignite.messaging.GridMessagingSelfTest;
+import org.apache.ignite.messaging.IgniteMessagingSendAsyncTest;
+import org.apache.ignite.messaging.IgniteMessagingWithClientTest;
+import org.apache.ignite.plugin.security.SecurityPermissionSetBuilderTest;
+import org.apache.ignite.spi.GridSpiLocalHostInjectionTest;
+import org.apache.ignite.startup.properties.NotStringSystemPropertyTest;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.test.ConfigVariationsTestSuiteBuilderTest;
+import org.apache.ignite.testframework.test.ParametersTest;
+import org.apache.ignite.testframework.test.VariationsIteratorTest;
+import org.apache.ignite.util.AttributeNodeFilterSelfTest;
+
+/**
+ *
+ */
+public class IgniteSanityTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        return suite(null);
+    }
+
+    /**
+     * @param ignoredTests Tests don't include in the execution.
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
+        TestSuite suite = new TestSuite("Ignite Sanity Suite");
+
+        suite.addTest(IgniteMarshallerSelfTestSuite.suite(ignoredTests));
+        suite.addTest(IgniteUtilSelfTestSuite.suite(ignoredTests));
+
+        suite.addTest(IgniteKernalSelfTestSuite.suite(ignoredTests));
+        suite.addTest(IgniteStartUpTestSuite.suite());
+        suite.addTest(IgniteExternalizableSelfTestSuite.suite());
+        suite.addTest(IgniteP2PSelfTestSuite.suite(ignoredTests));
+        suite.addTest(IgniteCacheP2pUnmarshallingErrorTestSuite.suite(ignoredTests));
+        suite.addTest(IgniteStreamSelfTestSuite.suite());
+
+        suite.addTest(IgnitePlatformsTestSuite.suite());
+
+        suite.addTest(new TestSuite(GridSelfTest.class));
+        suite.addTest(new TestSuite(ClusterGroupHostsSelfTest.class));
+        suite.addTest(new TestSuite(IgniteMessagingWithClientTest.class));
+        suite.addTest(new TestSuite(IgniteMessagingSendAsyncTest.class));
+
+        GridTestUtils.addTestIfNeeded(suite, ClusterGroupSelfTest.class, ignoredTests);
+        GridTestUtils.addTestIfNeeded(suite, GridMessagingSelfTest.class, ignoredTests);
+        GridTestUtils.addTestIfNeeded(suite, GridMessagingNoPeerClassLoadingSelfTest.class, ignoredTests);
+
+        if (U.isLinux() || U.isMacOs())
+            suite.addTest(IgniteIpcSharedMemorySelfTestSuite.suite());
+
+        GridTestUtils.addTestIfNeeded(suite, GridReleaseTypeSelfTest.class, ignoredTests);
+        suite.addTestSuite(GridProductVersionSelfTest.class);
+        suite.addTestSuite(GridAffinityProcessorRendezvousSelfTest.class);
+        suite.addTestSuite(GridClosureProcessorSelfTest.class);
+        suite.addTestSuite(GridClosureSerializationTest.class);
+        suite.addTestSuite(ClosureServiceClientsNodesTest.class);
+        suite.addTestSuite(GridStartStopSelfTest.class);
+        suite.addTestSuite(GridProjectionForCachesSelfTest.class);
+        suite.addTestSuite(GridProjectionForCachesOnDaemonNodeSelfTest.class);
+        suite.addTestSuite(GridSpiLocalHostInjectionTest.class);
+        suite.addTestSuite(GridLifecycleBeanSelfTest.class);
+        suite.addTestSuite(GridStopWithCancelSelfTest.class);
+        suite.addTestSuite(GridReduceSelfTest.class);
+        suite.addTestSuite(GridEventConsumeSelfTest.class);
+        suite.addTestSuite(GridSuppressedExceptionSelfTest.class);
+        suite.addTestSuite(GridLifecycleAwareSelfTest.class);
+        suite.addTestSuite(GridMessageListenSelfTest.class);
+        suite.addTestSuite(GridFailFastNodeFailureDetectionSelfTest.class);
+        suite.addTestSuite(OffHeapTieredTransactionSelfTest.class);
+        suite.addTestSuite(IgniteSlowClientDetectionSelfTest.class);
+        GridTestUtils.addTestIfNeeded(suite, IgniteDaemonNodeMarshallerCacheTest.class, ignoredTests);
+        suite.addTestSuite(IgniteMarshallerCacheConcurrentReadWriteTest.class);
+        suite.addTestSuite(GridNodeMetricsLogSelfTest.class);
+        suite.addTestSuite(GridLocalIgniteSerializationTest.class);
+
+        suite.addTestSuite(IgniteExceptionInNioWorkerSelfTest.class);
+        suite.addTestSuite(IgniteLocalNodeMapBeforeStartTest.class);
+        suite.addTestSuite(OdbcProcessorValidationSelfTest.class);
+        suite.addTestSuite(OdbcEscapeSequenceSelfTest.class);
+
+        GridTestUtils.addTestIfNeeded(suite, DynamicProxySerializationMultiJvmSelfTest.class, ignoredTests);
+
+        // Tests against configuration variations framework.
+        suite.addTestSuite(ParametersTest.class);
+        suite.addTestSuite(VariationsIteratorTest.class);
+        suite.addTestSuite(ConfigVariationsTestSuiteBuilderTest.class);
+        suite.addTestSuite(NotStringSystemPropertyTest.class);
+
+        suite.addTestSuite(MarshallerContextLockingSelfTest.class);
+        suite.addTestSuite(MarshallerContextSelfTest.class);
+
+        suite.addTestSuite(SecurityPermissionSetBuilderTest.class);
+
+        suite.addTestSuite(AttributeNodeFilterSelfTest.class);
+
+        return suite;
+    }
+}

From dbe82806695e2029e4e95d3e13e18642f0f383a5 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 22 Nov 2017 11:19:32 +0300
Subject: [PATCH 446/516] IGNITE-6984: Make cache creation slightly more
 verbose.

(cherry picked from commit bb89ad4)
---
 .../internal/processors/cache/GridCacheProcessor.java       | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index e9065603ed5ff..e1a497ffd14a4 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1146,7 +1146,11 @@ private void startCache(GridCacheAdapter<?, ?> cache) throws IgniteCheckedExcept
         cacheCtx.onStarted();
 
         if (log.isInfoEnabled())
-            log.info("Started cache [name=" + U.maskName(cfg.getName()) + ", mode=" + cfg.getCacheMode() + ']');
+            log.info("Started cache [name=" + U.maskName(cfg.getName()) +
+                ", id="+cacheCtx.cacheId() +
+                ", mode=" + cfg.getCacheMode() +
+                ", atomicity=" + cfg.getAtomicityMode() +
+                ", backups=" + cfg.getBackups() + ']');
     }
 
     /**

From a56a4110aebbdb12160ca2764915d71e854b2557 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 6 Jul 2017 14:53:25 +0300
Subject: [PATCH 447/516] Wait for condition should check condition if timeout
 detected.

---
 .../java/org/apache/ignite/testframework/GridTestUtils.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 0ae6575acc7c1..d5fb875e31c3d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -1490,7 +1490,7 @@ public static boolean waitForCondition(GridAbsPredicate cond, long timeout) thro
             curTime = U.currentTimeMillis();
         }
 
-        return false;
+        return cond.apply();
     }
 
     /**

From c1c7766c228fc41f8b73cff578f390fcd434c05c Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Thu, 6 Jul 2017 14:53:25 +0300
Subject: [PATCH 448/516] 2.1 Do print stack traces if tx is cancelled on node
 stop

(cherry picked from commit 2f1270e)
---
 .../processors/cache/GridCacheEntryEx.java    | 14 +---
 .../processors/cache/GridCacheMapEntry.java   | 72 ++++---------------
 .../GridDistributedTxRemoteAdapter.java       | 13 +++-
 .../dht/GridDhtTransactionalCacheAdapter.java |  2 +-
 .../near/GridNearTxFinishFuture.java          |  2 +-
 .../cache/transactions/IgniteTxAdapter.java   | 28 ++++----
 .../cache/transactions/IgniteTxHandler.java   |  2 +-
 .../transactions/IgniteTxLocalAdapter.java    | 12 +++-
 .../cache/GridCacheTestEntryEx.java           | 10 +--
 9 files changed, 56 insertions(+), 99 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index 6e011912f99b0..47fdbf5658674 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -201,23 +201,11 @@ public interface GridCacheEntryEx {
     /**
      * Invalidates this entry.
      *
-     * @param curVer Current version to match ({@code null} means always match).
      * @param newVer New version to set.
      * @return {@code true} if entry is obsolete.
      * @throws IgniteCheckedException If swap could not be released.
      */
-    public boolean invalidate(@Nullable GridCacheVersion curVer, GridCacheVersion newVer) throws IgniteCheckedException;
-
-    /**
-     * Invalidates this entry if it passes given filter.
-     *
-     * @param filter Optional filter that entry should pass before invalidation.
-     * @return {@code true} if entry was actually invalidated.
-     * @throws IgniteCheckedException If swap could not be released.
-     * @throws GridCacheEntryRemovedException If entry was removed.
-     */
-    public boolean invalidate(@Nullable CacheEntryPredicate[] filter)
-        throws GridCacheEntryRemovedException, IgniteCheckedException;
+    public boolean invalidate(GridCacheVersion newVer) throws IgniteCheckedException;
 
     /**
      * @param swap Swap flag.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index d395619d41899..ba7b3680d02dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -3057,32 +3057,30 @@ protected final boolean markObsolete0(GridCacheVersion ver, boolean clear, GridC
     }
 
     /** {@inheritDoc} */
-    @Override public synchronized boolean invalidate(@Nullable GridCacheVersion curVer, GridCacheVersion newVer)
+    @Override public synchronized boolean invalidate(GridCacheVersion newVer)
         throws IgniteCheckedException {
         assert newVer != null;
 
-        if (curVer == null || ver.equals(curVer)) {
-            CacheObject val = saveValueForIndexUnlocked();
+        CacheObject val = saveValueForIndexUnlocked();
 
-            value(null);
+        value(null);
 
-            ver = newVer;
-            flags &= ~IS_EVICT_DISABLED;
+        ver = newVer;
+        flags &= ~IS_EVICT_DISABLED;
 
-            if (log.isTraceEnabled()) {
-                log.trace("invalidate releaseSwap [key=" + key +
-                    ", entry=" + System.identityHashCode(this) +
-                    ", val=" + val +
-                    ", ptr=" + offHeapPointer() +
-                    ']');
-            }
+        if (log.isTraceEnabled()) {
+            log.trace("invalidate releaseSwap [key=" + key +
+                ", entry=" + System.identityHashCode(this) +
+                ", val=" + val +
+                ", ptr=" + offHeapPointer() +
+                ']');
+        }
 
-            releaseSwap();
+        releaseSwap();
 
-            clearIndex(val);
+        clearIndex(val);
 
-            onInvalidate();
-        }
+        onInvalidate();
 
         return obsoleteVersionExtras() != null;
     }
@@ -3094,46 +3092,6 @@ protected void onInvalidate() {
         // No-op.
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean invalidate(@Nullable CacheEntryPredicate[] filter)
-        throws GridCacheEntryRemovedException, IgniteCheckedException {
-        if (F.isEmptyOrNulls(filter)) {
-            synchronized (this) {
-                checkObsolete();
-
-                invalidate(null, nextVersion());
-
-                return true;
-            }
-        }
-        else {
-            // For optimistic checking.
-            GridCacheVersion startVer;
-
-            synchronized (this) {
-                checkObsolete();
-
-                startVer = ver;
-            }
-
-            if (!cctx.isAll(this, filter))
-                return false;
-
-            synchronized (this) {
-                checkObsolete();
-
-                if (startVer.equals(ver)) {
-                    invalidate(null, nextVersion());
-
-                    return true;
-                }
-            }
-
-            // If version has changed then repeat the process.
-            return invalidate(filter);
-        }
-    }
-
     /**
      *
      * @param val New value.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index 68c0e575d59f3..fc28c1f1d92d5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -56,6 +57,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -694,14 +696,19 @@ else if (op == READ) {
                         }
                     }
                     catch (Throwable ex) {
-                        // In case of error, we still make the best effort to commit,
+                        boolean nodeStopping = X.hasCause(ex, NodeStoppingException.class);// In case of error, we still make the best effort to commit,
                         // as there is no way to rollback at this point.
                         err = new IgniteTxHeuristicCheckedException("Commit produced a runtime exception " +
                             "(all transaction entries will be invalidated): " + CU.txString(this), ex);
 
-                        U.error(log, "Commit failed.", err);
+                        if (nodeStopping) {
+                            U.warn(log, "Failed to commit transaction, node is stopping [tx=" + this +
+                                ", err=" + ex + ']');
+                        }
+                        else
+                            U.error(log, "Commit failed.", err);
 
-                        uncommit();
+                        uncommit(nodeStopping);
 
                         state(UNKNOWN);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index 4f8d73cb4f22f..36964575836d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -1710,7 +1710,7 @@ private void invalidateNearEntry(KeyCacheObject key, GridCacheVersion ver) throw
         GridCacheEntryEx nearEntry = near().peekEx(key);
 
         if (nearEntry != null)
-            nearEntry.invalidate(null, ver);
+            nearEntry.invalidate(ver);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 512f63e097cd9..903bb8f1e9368 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -354,7 +354,7 @@ else if (err != null)
                                     GridCacheEntryEx entry = cacheCtx.cache().peekEx(e.key());
 
                                     if (entry != null)
-                                        entry.invalidate(null, tx.xidVersion());
+                                        entry.invalidate(tx.xidVersion());
                                 }
                             }
                             catch (Throwable t) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 266c5a83cd8c9..1a84a78ab6371 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -429,23 +429,27 @@ public void storeEnabled(boolean storeEnabled) {
 
     /**
      * Uncommits transaction by invalidating all of its entries. Courtesy to minimize inconsistency.
+     *
+     * @param nodeStopping {@code True} if tx was cancelled during node stop.
      */
     @SuppressWarnings({"CatchGenericClass"})
-    protected void uncommit() {
-        for (IgniteTxEntry e : writeMap().values()) {
-            try {
-                GridCacheEntryEx Entry = e.cached();
+    protected void uncommit(boolean nodeStopping) {
+        if (!nodeStopping) {
+            for (IgniteTxEntry e : writeMap().values()) {
+                try {
+                    GridCacheEntryEx entry = e.cached();
 
-                if (e.op() != NOOP)
-                    Entry.invalidate(null, xidVer);
-            }
-            catch (Throwable t) {
-                U.error(log, "Failed to invalidate transaction entries while reverting a commit.", t);
+                    if (e.op() != NOOP)
+                        entry.invalidate(xidVer);
+                }
+                catch (Throwable t) {
+                    U.error(log, "Failed to invalidate transaction entries while reverting a commit.", t);
 
-                if (t instanceof Error)
-                    throw (Error)t;
+                    if (t instanceof Error)
+                        throw (Error)t;
 
-                break;
+                    break;
+                }
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 9c7dfd579309c..09392e0d849c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -1624,7 +1624,7 @@ private void invalidateNearEntry(GridCacheContext cacheCtx, KeyCacheObject key,
         GridCacheEntryEx nearEntry = near.peekEx(key);
 
         if (nearEntry != null)
-            nearEntry.invalidate(null, ver);
+            nearEntry.invalidate(ver);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 50426a1fe72e8..92893c15464c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -37,6 +37,7 @@
 import javax.cache.processor.EntryProcessor;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
@@ -956,11 +957,18 @@ assert ownsLock(txEntry.cached()):
                             throw ex;
                         }
                         else {
+                            boolean nodeStopping = X.hasCause(ex, NodeStoppingException.class);
+
                             IgniteCheckedException err = new IgniteTxHeuristicCheckedException("Failed to locally write to cache " +
                                 "(all transaction entries will be invalidated, however there was a window when " +
                                 "entries for this transaction were visible to others): " + this, ex);
 
-                            U.error(log, "Heuristic transaction failure.", err);
+                            if (nodeStopping) {
+                                U.warn(log, "Failed to commit transaction, node is stopping " +
+                                    "[tx=" + this + ", err=" + ex + ']');
+                            }
+                            else
+                                U.error(log, "Heuristic transaction failure.", err);
 
                             COMMIT_ERR_UPD.compareAndSet(this, null, err);
 
@@ -968,7 +976,7 @@ assert ownsLock(txEntry.cached()):
 
                             try {
                                 // Courtesy to minimize damage.
-                                uncommit();
+                                uncommit(nodeStopping);
                             }
                             catch (Throwable ex1) {
                                 U.error(log, "Failed to uncommit transaction: " + this, ex1);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 1ecc2d15670a8..de54ad6dea533 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -358,21 +358,13 @@ void recheckLock() {
     }
 
     /** @inheritDoc */
-    @Override public boolean invalidate(@Nullable GridCacheVersion curVer, GridCacheVersion newVer)
+    @Override public boolean invalidate(GridCacheVersion newVer)
         throws IgniteCheckedException {
         assert false;
 
         return false;
     }
 
-    /** @inheritDoc */
-    @Override public boolean invalidate(@Nullable CacheEntryPredicate[] filter)
-        throws GridCacheEntryRemovedException, IgniteCheckedException {
-        assert false;
-
-        return false;
-    }
-
     /** @inheritDoc */
     @Override public boolean evictInternal(boolean swap, GridCacheVersion obsoleteVer,
         @Nullable CacheEntryPredicate[] filter) {

From 7441f334680ff30be2f8ac9562ebf7683fd3cb31 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Mon, 9 Jan 2017 11:42:39 +0300
Subject: [PATCH 449/516] Fixed IGNITE-6838. Restore EvictionPolicy 'maxSize'
 field default value.

---
 .../ignite/cache/eviction/AbstractEvictionPolicyFactory.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicyFactory.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicyFactory.java
index 012c7ee522ea9..aa7dea97728a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicyFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicyFactory.java
@@ -20,12 +20,14 @@
 import javax.cache.configuration.Factory;
 import org.apache.ignite.internal.util.typedef.internal.A;
 
+import static org.apache.ignite.configuration.CacheConfiguration.DFLT_CACHE_SIZE;
+
 /**
  * Common functionality implementation for eviction policies factories.
  */
 public abstract class AbstractEvictionPolicyFactory<T> implements Factory<T> {
     /** */
-    private int maxSize;
+    private int maxSize = DFLT_CACHE_SIZE;
 
     /** */
     private int batchSize = 1;

From 67cb0db4b4cefc8edde39f59c81a3f510dee2bfd Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 4 Dec 2017 11:13:23 +0300
Subject: [PATCH 450/516] Mute tests: GG-13126, GG-13125

(cherry picked from commit 6c1b988)
---
 .../processors/cache/GridCacheAbstractFullApiSelfTest.java     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
index ba75dcb714179..6ec06398afd77 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
@@ -2741,6 +2741,9 @@ public void testDeletedEntriesFlag() throws Exception {
      * @throws Exception If failed.
      */
     public void testRemoveLoad() throws Exception {
+        if (memoryMode() == OFFHEAP_TIERED)
+            fail("https://ggsystems.atlassian.net/browse/GG-13126");
+
         int cnt = 10;
 
         Set<String> keys = new HashSet<>();

From 1d4ab0fa4d3bd363f450e085b21c9c3aec7cf02e Mon Sep 17 00:00:00 2001
From: apopov <tank2.alex@gmail.com>
Date: Thu, 7 Dec 2017 13:11:09 +0300
Subject: [PATCH 451/516] GG-13134: backward compatibility issue for
 GridDhtForceKeysResponse

---
 .../distributed/dht/preloader/GridDhtForceKeysResponse.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java
index 8d90158700075..4a84bf1f5dc50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java
@@ -171,7 +171,8 @@ public void addInfo(GridCacheEntryInfo info) {
                 info.marshal(cctx);
         }
 
-        if (err != null && errBytes == null)
+        // GG-13134: we should always marshall err (even if err == null) for backward compatibility
+        if (errBytes == null)
             errBytes = U.marshal(ctx, err);
     }
 

From 7950a7850b769483ec2994fa8445da521efdfa4a Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 8 Dec 2017 13:36:28 +0300
Subject: [PATCH 452/516] IGNITE-7086 - Backups are not updated when
 ReadFromBackup=true and ReadThrough happens.

(cherry picked from commit c769838)
---
 .../processors/cache/GridCacheUtils.java      | 106 +++++++-
 .../dht/GridPartitionedGetFuture.java         |  28 ++-
 .../dht/GridPartitionedSingleGetFuture.java   |  25 ++
 .../distributed/near/GridNearGetFuture.java   |  32 ++-
 .../store/CacheStoreReadFromBackupTest.java   | 238 ++++++++++++++++++
 ...eTransactionalStoreReadFromBackupTest.java |  32 +++
 .../testsuites/IgniteCacheTestSuite.java      |   4 +
 7 files changed, 457 insertions(+), 8 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreReadFromBackupTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 2a976ecbe230c..171be1e362918 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -62,11 +62,15 @@
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.dr.GridDrType;
 import org.apache.ignite.internal.processors.igfs.IgfsUtils;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.util.lang.IgniteInClosureX;
@@ -78,6 +82,7 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -97,7 +102,6 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheMode.LOCAL;
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
 import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
@@ -1721,4 +1725,104 @@ public static TransactionConfiguration transactionConfiguration(final @Nullable
             ? DEFAULT_TX_CFG
             : cfg.getTransactionConfiguration();
     }
+
+    /**
+     * Creates closure that saves initial value to backup partition.
+     * <p>
+     * Useful only when store with readThrough is used. In situation when
+     * get() on backup node returns successful result, it's expected that
+     * localPeek() will be successful as well. But it doesn't true when
+     * primary node loaded value from local store, in this case backups
+     * will remain non-initialized.
+     * <br>
+     * To meet that requirement the value requested from primary should
+     * be saved on backup during get().
+     * </p>
+     *
+     * @param topVer Topology version.
+     * @param log Logger.
+     * @param cctx Cache context.
+     * @param key Key.
+     * @param readThrough Read through.
+     * @param skipVals Skip values.
+     */
+    @Nullable public static BackupPostProcessingClosure createBackupPostProcessingClosure(
+        final AffinityTopologyVersion topVer,
+        final IgniteLogger log,
+        final GridCacheContext cctx,
+        final @Nullable KeyCacheObject key,
+        boolean readThrough,
+        boolean skipVals
+    ) {
+        if (!readThrough || skipVals ||
+            (key != null && !cctx.affinity().backupsByKey(key, topVer).contains(cctx.localNode())))
+            return null;
+
+        return new BackupPostProcessingClosure() {
+            private void process(KeyCacheObject key, CacheObject val, GridCacheVersion ver, GridDhtCacheAdapter colocated) {
+                while (true) {
+                    GridCacheEntryEx entry = null;
+
+                    try {
+                        entry = colocated.entryEx(key, topVer);
+
+                        entry.initialValue(
+                            val,
+                            ver,
+                            0,
+                            0,
+                            false,
+                            topVer,
+                            GridDrType.DR_BACKUP,
+                            true);
+
+                        break;
+                    }
+                    catch (GridCacheEntryRemovedException ignore) {
+                        if (log.isDebugEnabled())
+                            log.debug("Got removed entry during postprocessing (will retry): " +
+                                entry);
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Error saving backup value: " + entry, e);
+                    }
+                    catch (GridDhtInvalidPartitionException ignored) {
+                        break;
+                    }
+                    finally {
+                        if (entry != null)
+                            cctx.evicts().touch(entry, topVer);
+                    }
+                }
+            }
+
+            @Override public void apply(CacheObject val, GridCacheVersion ver) {
+                process(key, val, ver, cctx.dht());
+            }
+
+            @Override public void apply(Collection<GridCacheEntryInfo> infos) {
+                if (!F.isEmpty(infos)) {
+                    GridCacheAffinityManager aff = cctx.affinity();
+                    ClusterNode locNode = cctx.localNode();
+
+                    GridDhtCacheAdapter colocated = cctx.cache().isNear()
+                        ? ((GridNearCacheAdapter)cctx.cache()).dht()
+                        : cctx.dht();
+
+                    for (GridCacheEntryInfo info : infos) {
+                        // Save backup value.
+                        if (aff.backupsByKey(info.key(), topVer).contains(locNode))
+                            process(info.key(), info.value(), info.version(), colocated);
+                    }
+                }
+            }
+        };
+    }
+
+    /**
+     *
+     */
+    public interface BackupPostProcessingClosure extends IgniteInClosure<Collection<GridCacheEntryInfo>>,
+        IgniteBiInClosure<CacheObject, GridCacheVersion>{
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 4dcb300e92fb1..560bf185f062a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
 
@@ -234,7 +235,7 @@ private boolean isMini(IgniteInternalFuture<?> f) {
     private void map(
         Collection<KeyCacheObject> keys,
         Map<ClusterNode, LinkedHashMap<KeyCacheObject, Boolean>> mapped,
-        AffinityTopologyVersion topVer
+        final AffinityTopologyVersion topVer
     ) {
         Collection<ClusterNode> cacheNodes = CU.affinityNodes(cctx, topVer);
 
@@ -348,7 +349,8 @@ private void map(
                 }));
             }
             else {
-                MiniFuture fut = new MiniFuture(n, mappedKeys, topVer);
+                MiniFuture fut = new MiniFuture(n, mappedKeys, topVer,
+                    CU.createBackupPostProcessingClosure(topVer, log, cctx, null, readThrough, skipVals));
 
                 GridCacheMessage req = new GridNearGetRequest(
                     cctx.cacheId(),
@@ -676,6 +678,9 @@ private class MiniFuture extends GridFutureAdapter<Map<K, V>> {
         /** Topology version on which this future was mapped. */
         private final AffinityTopologyVersion topVer;
 
+        /** Post processing closure. */
+        private final IgniteInClosure<Collection<GridCacheEntryInfo>> postProcessingClos;
+
         /** {@code True} if remapped after node left. */
         private boolean remapped;
 
@@ -683,11 +688,14 @@ private class MiniFuture extends GridFutureAdapter<Map<K, V>> {
          * @param node Node.
          * @param keys Keys.
          * @param topVer Topology version.
+         * @param postProcessingClos Post processing closure.
          */
-        MiniFuture(ClusterNode node, LinkedHashMap<KeyCacheObject, Boolean> keys, AffinityTopologyVersion topVer) {
+        MiniFuture(ClusterNode node, LinkedHashMap<KeyCacheObject, Boolean> keys, AffinityTopologyVersion topVer,
+            @Nullable IgniteInClosure<Collection<GridCacheEntryInfo>> postProcessingClos) {
             this.node = node;
             this.keys = keys;
             this.topVer = topVer;
+            this.postProcessingClos = postProcessingClos;
         }
 
         /**
@@ -805,6 +813,8 @@ void onResult(final GridNearGetResponse res) {
                         }
                     }), F.t(node, keys), topVer);
 
+                    postProcessResult(res);
+
                     onDone(createResultMap(res.entries()));
 
                     return;
@@ -825,12 +835,16 @@ void onResult(final GridNearGetResponse res) {
                             }
                         }), F.t(node, keys), topVer);
 
+                        postProcessResult(res);
+
                         onDone(createResultMap(res.entries()));
                     }
                 });
             }
             else {
                 try {
+                    postProcessResult(res);
+
                     onDone(createResultMap(res.entries()));
                 }
                 catch (Exception e) {
@@ -839,6 +853,14 @@ void onResult(final GridNearGetResponse res) {
             }
         }
 
+        /**
+         * @param res Response.
+         */
+        private void postProcessResult(final GridNearGetResponse res) {
+            if (postProcessingClos != null)
+                postProcessingClos.apply(res.entries());
+        }
+
         /** {@inheritDoc} */
         @Override public String toString() {
             return S.toString(MiniFuture.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index b667a10f889d2..cda7d5201e596 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheMessage;
 import org.apache.ignite.internal.processors.cache.GridCacheSwapEntry;
+import org.apache.ignite.internal.processors.cache.GridCacheUtils.BackupPostProcessingClosure;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.near.CacheVersionedValue;
@@ -52,6 +53,7 @@
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteProductVersion;
@@ -130,6 +132,9 @@ public class GridPartitionedSingleGetFuture extends GridFutureAdapter<Object> im
     @GridToStringInclude
     private ClusterNode node;
 
+    /** Post processing closure. */
+    private volatile BackupPostProcessingClosure postProcessingClos;
+
     /**
      * @param cctx Context.
      * @param key Key.
@@ -286,6 +291,17 @@ private void map(AffinityTopologyVersion topVer) {
                 cctx.mvcc().addFuture(this, futId);
             }
 
+            boolean needVer = this.needVer;
+
+            final BackupPostProcessingClosure postClos = CU.createBackupPostProcessingClosure(topVer, log, cctx, key, readThrough, skipVals);
+
+            if (postClos != null) {
+                // Need version to correctly store value.
+                needVer = true;
+
+                postProcessingClos = postClos;
+            }
+
             GridCacheMessage req;
 
             if (node.version().compareTo(SINGLE_GET_MSG_SINCE) >= 0) {
@@ -541,6 +557,12 @@ public void onResult(UUID nodeId, GridNearSingleGetResponse res) {
         else {
             if (skipVals)
                 setSkipValueResult(res.containsValue(), null);
+            else if (readThrough && res0 instanceof CacheVersionedValue) {
+                // Could be versioned value for store in backup.
+                CacheVersionedValue verVal = (CacheVersionedValue)res0;
+
+                setResult(verVal.value(), verVal.version());
+            }
             else
                 setResult((CacheObject)res0, null);
         }
@@ -679,6 +701,9 @@ private void setResult(@Nullable CacheObject val, @Nullable GridCacheVersion ver
             assert !skipVals;
 
             if (val != null) {
+                if (postProcessingClos != null)
+                    postProcessingClos.apply(val, ver);
+
                 if (!keepCacheObjects) {
                     Object res = cctx.unwrapBinaryIfNeeded(val, !deserializeBinary);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index b061593b109fa..95ad6d82222c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
 import org.apache.ignite.internal.processors.cache.GridCacheMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheUtils.BackupPostProcessingClosure;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.dht.CacheDistributedGetFutureAdapter;
@@ -378,7 +379,8 @@ private void map(
                     cctx.mvcc().addFuture(this, futId);
                 }
 
-                MiniFuture fut = new MiniFuture(n, mappedKeys, saved, topVer);
+                MiniFuture fut = new MiniFuture(n, mappedKeys, saved, topVer,
+                    CU.createBackupPostProcessingClosure(topVer, log, cctx, null, readThrough, skipVals));
 
                 GridCacheMessage req = new GridNearGetRequest(
                     cctx.cacheId(),
@@ -411,6 +413,8 @@ private void map(
         }
     }
 
+
+
     /**
      * @param mappings Mappings.
      * @param key Key to map.
@@ -858,6 +862,9 @@ private class MiniFuture extends GridFutureAdapter<Map<K, V>> {
         /** Topology version on which this future was mapped. */
         private AffinityTopologyVersion topVer;
 
+        /** Post processing closure. */
+        private final BackupPostProcessingClosure postProcessingClos;
+
         /** {@code True} if remapped after node left. */
         private boolean remapped;
 
@@ -866,17 +873,19 @@ private class MiniFuture extends GridFutureAdapter<Map<K, V>> {
          * @param keys Keys.
          * @param savedEntries Saved entries.
          * @param topVer Topology version.
+         * @param postProcessingClos Post processing closure.
          */
         MiniFuture(
             ClusterNode node,
             LinkedHashMap<KeyCacheObject, Boolean> keys,
             Map<KeyCacheObject, GridNearCacheEntry> savedEntries,
-            AffinityTopologyVersion topVer
-        ) {
+            AffinityTopologyVersion topVer,
+            BackupPostProcessingClosure postProcessingClos) {
             this.node = node;
             this.keys = keys;
             this.savedEntries = savedEntries;
             this.topVer = topVer;
+            this.postProcessingClos = postProcessingClos;
         }
 
         /**
@@ -1003,6 +1012,8 @@ void onResult(final GridNearGetResponse res) {
                         }
                     }), F.t(node, keys), topVer);
 
+                    postProcessResult(res);
+
                     // It is critical to call onDone after adding futures to compound list.
                     onDone(loadEntries(node.id(), keys.keySet(), res.entries(), savedEntries, topVer));
 
@@ -1023,13 +1034,26 @@ void onResult(final GridNearGetResponse res) {
                             }
                         }), F.t(node, keys), readyTopVer);
 
+                        postProcessResult(res);
+
                         // It is critical to call onDone after adding futures to compound list.
                         onDone(loadEntries(node.id(), keys.keySet(), res.entries(), savedEntries, topVer));
                     }
                 });
             }
-            else
+            else {
+                postProcessResult(res);
+
                 onDone(loadEntries(node.id(), keys.keySet(), res.entries(), savedEntries, topVer));
+            }
+        }
+
+        /**
+         * @param res Response.
+         */
+        private void postProcessResult(final GridNearGetResponse res) {
+            if (postProcessingClos != null)
+                postProcessingClos.apply(res.entries());
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreReadFromBackupTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreReadFromBackupTest.java
new file mode 100644
index 0000000000000..d8913dcf1992f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreReadFromBackupTest.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.cache.Cache;
+import javax.cache.configuration.FactoryBuilder;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.lang.IgniteBiInClosure;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CachePeekMode.BACKUP;
+import static org.apache.ignite.cache.CachePeekMode.PRIMARY;
+
+/**
+ * Checks that once value is read from store, it will be loaded in
+ * backups as well.
+ */
+public class CacheStoreReadFromBackupTest extends GridCommonAbstractTest {
+    /** */
+    public static final String CACHE_NAME = "cache";
+
+    /** */
+    private static final Map<Integer, String> storeMap = new ConcurrentHashMap<>();
+
+    /** */
+    private CacheMode cacheMode = REPLICATED;
+
+    /** */
+    private int backups;
+
+    /** Near. */
+    private boolean near;
+
+    /** */
+    @SuppressWarnings("unchecked")
+    private CacheConfiguration cacheConfig(String cacheName) {
+        CacheConfiguration ccfg = new CacheConfiguration<>(cacheName);
+
+        ccfg.setCacheMode(cacheMode);
+        ccfg.setBackups(backups);
+        ccfg.setAtomicityMode(atomicityMode());
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 1));
+        ccfg.setReadThrough(true);
+        ccfg.setReadFromBackup(true);
+        ccfg.setCacheStoreFactory(FactoryBuilder.factoryOf(TestStore.class));
+
+        if (near)
+            ccfg.setNearConfiguration(new NearCacheConfiguration());
+
+        return ccfg;
+    }
+
+    /** */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        return super.getConfiguration(gridName).setCacheConfiguration(cacheConfig(CACHE_NAME));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @return Atomicity mode.
+     */
+    protected CacheAtomicityMode atomicityMode() {
+        return ATOMIC;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReplicated() throws Exception {
+        cacheMode = REPLICATED;
+        backups = 0;
+        near = false;
+
+        checkReadFromBackup();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPartitioned() throws Exception {
+        cacheMode = PARTITIONED;
+        backups = 1;
+        near = false;
+
+        checkReadFromBackup();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNearReplicated() throws Exception {
+        cacheMode = REPLICATED;
+        backups = 0;
+        near = true;
+
+        checkReadFromBackup();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNearPartitioned() throws Exception {
+        cacheMode = PARTITIONED;
+        backups = 1;
+        near = true;
+
+        checkReadFromBackup();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkReadFromBackup() throws Exception {
+        startGridsMultiThreaded(2, true);
+
+        checkReadSingleFromBackup();
+        checkReadAllFromBackup();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkReadSingleFromBackup() throws Exception {
+        storeMap.put(1, "val-1");
+
+        IgniteCache<Integer, String> cache0 = grid(0).cache(CACHE_NAME);
+        IgniteCache<Integer, String> cache1 = grid(1).cache(CACHE_NAME);
+
+        // Load value on primary and backup.
+        assertNotNull(cache0.get(1));
+        assertNotNull(cache1.get(1));
+
+        if (cache0.localPeek(1, PRIMARY) != null)
+            assertNotNull(cache1.localPeek(1, BACKUP));
+        else {
+            assertNotNull(cache0.localPeek(1, BACKUP));
+            assertNotNull(cache1.localPeek(1, PRIMARY));
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkReadAllFromBackup() throws Exception {
+        for (int i = 0; i < 100; i++)
+            storeMap.put(i, String.valueOf(i));
+
+        IgniteCache<Integer, String> cache0 = grid(0).cache(CACHE_NAME);
+        IgniteCache<Integer, String> cache1 = grid(1).cache(CACHE_NAME);
+
+        assertEquals(storeMap.size(), cache0.getAll(storeMap.keySet()).size());
+        assertEquals(storeMap.size(), cache1.getAll(storeMap.keySet()).size());
+
+        Affinity<Integer> aff = grid(0).affinity(CACHE_NAME);
+        ClusterNode node0 = grid(0).cluster().localNode();
+
+        for (Integer key : storeMap.keySet()) {
+            if (aff.isPrimary(node0, key)) {
+                assertNotNull(cache0.localPeek(key, PRIMARY));
+                assertNotNull(cache1.localPeek(key, BACKUP));
+            }
+            else {
+                assertNotNull(cache0.localPeek(key, BACKUP));
+                assertNotNull(cache1.localPeek(key, PRIMARY));
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    public static class TestStore extends CacheStoreAdapter<Integer, String> {
+        /** */
+        public TestStore() {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void loadCache(IgniteBiInClosure<Integer, String> clo, Object... args) {
+            for (Map.Entry<Integer, String> e : storeMap.entrySet())
+                clo.apply(e.getKey(), e.getValue());
+        }
+
+        /** {@inheritDoc} */
+        @Override public String load(Integer key) {
+            return storeMap.get(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<? extends Integer, ? extends String> entry) {
+            storeMap.put(entry.getKey(), entry.getValue());
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("SuspiciousMethodCalls")
+        @Override public void delete(Object key) {
+            storeMap.remove(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sessionEnd(boolean commit) {
+            // No-op.
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java
new file mode 100644
index 0000000000000..4837936621f46
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+
+/**
+ *
+ */
+public class CacheTransactionalStoreReadFromBackupTest extends CacheStoreReadFromBackupTest {
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return TRANSACTIONAL;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index fd8d43fa46d7a..e2ab118bbc88a 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -29,6 +29,8 @@
 import org.apache.ignite.cache.affinity.fair.FairAffinityFunctionSelfTest;
 import org.apache.ignite.cache.affinity.fair.FairAffinityNodesRestart;
 import org.apache.ignite.cache.affinity.local.LocalAffinityFunctionTest;
+import org.apache.ignite.cache.store.CacheStoreReadFromBackupTest;
+import org.apache.ignite.cache.store.CacheTransactionalStoreReadFromBackupTest;
 import org.apache.ignite.cache.store.GridCacheBalancingStoreSelfTest;
 import org.apache.ignite.cache.store.GridCacheLoadOnlyStoreAdapterSelfTest;
 import org.apache.ignite.cache.store.GridStoreLoadCacheTest;
@@ -356,6 +358,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(IgniteIoTestMessagesTest.class);
 
         suite.addTestSuite(GridStoreLoadCacheTest.class);
+        suite.addTestSuite(CacheStoreReadFromBackupTest.class);
+        suite.addTestSuite(CacheTransactionalStoreReadFromBackupTest.class);
 
         return suite;
     }

From 7614545f081653a9f79c172802c48e3759a416b1 Mon Sep 17 00:00:00 2001
From: Yakov Zhdanov <yzhdanov@gridgain.com>
Date: Thu, 2 Mar 2017 15:01:50 +0300
Subject: [PATCH 453/516] put all benchmark - added striping and pregenerated
 maps

(cherry picked from commit 76cd44d6fc8a195b2a20016c144836e684330b40)
---
 .../yardstick/IgniteAbstractBenchmark.java    |  6 ++
 .../yardstick/IgniteBenchmarkArguments.java   | 11 +++
 .../cache/IgnitePutAllBenchmark.java          | 95 +++++++++++++++++--
 .../cache/IgnitePutAllTxBenchmark.java        | 43 +--------
 4 files changed, 103 insertions(+), 52 deletions(-)

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
index 522499ac96143..a221e5ca2cfdc 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
@@ -20,6 +20,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteState;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.events.Event;
@@ -61,6 +62,11 @@ public abstract class IgniteAbstractBenchmark extends BenchmarkDriverAdapter {
             node = new IgniteNode(args.isClientOnly() && !args.isNearCache(), Ignition.ignite());
 
         waitForNodes();
+
+        IgniteLogger log = ignite().log();
+
+        if (log.isInfoEnabled())
+            log.info("Benchmark arguments: " + args);
     }
 
     /**
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 2d2da5a27db8f..23502083ce5a8 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -124,6 +124,10 @@ public class IgniteBenchmarkArguments {
     @Parameter(names = {"-col", "--collocated"}, description = "Collocated")
     private boolean collocated;
 
+    /** */
+    @Parameter(names = {"-stripe", "--singleStripe"}, description = "Generate keys belonging to single stripe per node")
+    private boolean singleStripe;
+
     /** */
     @Parameter(names = {"-jdbc", "--jdbcUrl"}, description = "JDBC url")
     private String jdbcUrl;
@@ -349,6 +353,13 @@ public boolean collocated() {
         return collocated;
     }
 
+    /**
+     * @return Generate keys for single stripe per node.
+     */
+    public boolean singleStripe() {
+        return singleStripe;
+    }
+
     /**
      * @return Delay in second which used in nodes restart algorithm.
      */
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllBenchmark.java
index 8cd2347889338..33aa2059bbc8f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllBenchmark.java
@@ -17,43 +17,118 @@
 
 package org.apache.ignite.yardstick.cache;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
 import java.util.Map;
-import java.util.SortedMap;
 import java.util.TreeMap;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.yardstickframework.BenchmarkConfiguration;
 
 /**
  * Ignite benchmark that performs putAll operations.
  */
 public class IgnitePutAllBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
+    /** */
+    private static final Integer PUT_MAPS_KEY = 2048;
+
+    /** */
+    private static final Integer PUT_MAPS_CNT = 256;
+
     /** Affinity mapper. */
     private Affinity<Integer> aff;
 
+    /** */
+    private int srvrCnt;
+
+    /** */
+    private int stripesCnt;
+
     /** {@inheritDoc} */
     @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
         super.setUp(cfg);
 
-        aff = ignite().affinity("atomic");
+        aff = ignite().affinity(cache().getName());
+
+        Collection<ClusterNode> nodes = ignite().cluster().forServers().nodes();
+
+        stripesCnt = ignite().cluster().forServers().forRandom().metrics().getTotalCpus();
+
+        srvrCnt = nodes.size();
+
+        IgniteLogger log = ignite().log();
+
+        if (log.isInfoEnabled())
+            log.info("Servers info [srvrsCnt=" + srvrCnt + ", stripesCnt=" + stripesCnt + ']');
     }
 
     /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
-        SortedMap<Integer, Integer> vals = new TreeMap<>();
+        List<Map<Integer, Integer>> putMaps = (List<Map<Integer, Integer>>)ctx.get(PUT_MAPS_KEY);
+
+        if (putMaps == null) {
+            putMaps = new ArrayList<>(PUT_MAPS_CNT);
+
+            ctx.put(PUT_MAPS_KEY, putMaps);
+        }
+
+        Map<Integer, Integer> vals;
+
+        if (putMaps.size() == PUT_MAPS_CNT)
+            vals = putMaps.get(nextRandom(PUT_MAPS_CNT));
+        else {
+            vals = new TreeMap<>();
+
+            ClusterNode node = args.collocated() ? aff.mapKeyToNode(nextRandom(args.range())) : null;
+
+            Map<ClusterNode, Integer> stripesMap = null;
+
+            if (args.singleStripe())
+                stripesMap = U.newHashMap(srvrCnt);
+
+            for (; vals.size() < args.batch(); ) {
+                int key = nextRandom(args.range());
+
+                if (args.collocated() && !aff.isPrimary(
+                    node,
+                    key))
+                    continue;
+
+                if (args.singleStripe()) {
+                    int part = aff.partition(key);
+
+                    ClusterNode node0 = node != null ? node : aff.mapPartitionToNode(part);
+
+                    Integer stripe0 = stripesMap.get(node0);
+                    int stripe = part % stripesCnt;
 
-        ClusterNode node = args.collocated() ? aff.mapKeyToNode(nextRandom(args.range())) : null;
+                    if (stripe0 != null) {
+                        if (stripe0 != stripe)
+                            continue;
+                    }
+                    else
+                        stripesMap.put(
+                            node0,
+                            stripe);
+                }
 
-        for (int i = 0; i < args.batch(); ) {
-            int key = nextRandom(args.range());
+                vals.put(
+                    key,
+                    key);
+            }
 
-            if (args.collocated() && !aff.isPrimary(node, key))
-                continue;
+            putMaps.add(vals);
 
-            ++i;
+            if (putMaps.size() == PUT_MAPS_CNT) {
+                IgniteLogger log = ignite().log();
 
-            vals.put(key, key);
+                if (log.isInfoEnabled())
+                    log.info("Put maps set generated.");
+            }
         }
 
         cache.putAll(vals);
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxBenchmark.java
index 63faa2f35403a..d701314cd3078 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxBenchmark.java
@@ -17,53 +17,12 @@
 
 package org.apache.ignite.yardstick.cache;
 
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cluster.ClusterNode;
-import org.yardstickframework.BenchmarkConfiguration;
 
 /**
  * Ignite benchmark that performs transactional putAll operations.
  */
-public class IgnitePutAllTxBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
-    /** Affinity mapper. */
-    private Affinity<Integer> aff;
-
-    /** {@inheritDoc} */
-    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
-        super.setUp(cfg);
-
-        aff = ignite().affinity("tx");
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
-        ThreadRange r = threadRange();
-
-        SortedMap<Integer, Integer> vals = new TreeMap<>();
-
-        ClusterNode node = args.collocated() ? aff.mapKeyToNode(r.nextRandom()) : null;
-
-        for (int i = 0; i < args.batch(); ) {
-            int key = r.nextRandom();
-
-            if (args.collocated() && !aff.isPrimary(node, key))
-                continue;
-
-            ++i;
-
-            vals.put(key, key);
-        }
-
-        // Implicit transaction is used.
-        cache.putAll(vals);
-
-        return true;
-    }
-
+public class IgnitePutAllTxBenchmark extends IgnitePutAllBenchmark {
     /** {@inheritDoc} */
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("tx");

From ded0fea27ef45a0dcb1a26147c10c9a61dc79636 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 7 Dec 2017 19:16:25 +0300
Subject: [PATCH 454/516] Backport of IGNITE-7008: TcpDiscoverySharedFsIpFinder
 fails with NPE if address can't be resolved. This closes #3087.

Signed-off-by: nikolay_tikhonov <ntikhonov@gridgain.com>

(cherry picked from commit e39283e)
---
 .../TcpDiscoverySharedFsIpFinder.java         | 34 ++++++++-----------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/sharedfs/TcpDiscoverySharedFsIpFinder.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/sharedfs/TcpDiscoverySharedFsIpFinder.java
index d4e93d2de948f..1ff15b6c746e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/sharedfs/TcpDiscoverySharedFsIpFinder.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/sharedfs/TcpDiscoverySharedFsIpFinder.java
@@ -183,29 +183,24 @@ private File initFolder() throws IgniteSpiException {
 
         Collection<InetSocketAddress> addrs = new LinkedList<>();
 
-        for (String fileName : folder.list())
-            if (!".svn".equals(fileName)) {
-                InetSocketAddress addr = null;
+        for (String fileName : folder.list()) {
+            StringTokenizer st = new StringTokenizer(fileName, DELIM);
 
-                StringTokenizer st = new StringTokenizer(fileName, DELIM);
+            if (st.countTokens() != 2)
+                continue;
 
-                if (st.countTokens() == 2) {
-                    String addrStr = st.nextToken();
-                    String portStr = st.nextToken();
+            String addrStr = st.nextToken();
+            String portStr = st.nextToken();
 
-                    try {
-                        int port = Integer.parseInt(portStr);
-
-                        addr = new InetSocketAddress(denormalizeAddress(addrStr), port);
-                    }
-                    catch (IllegalArgumentException e) {
-                        U.error(log, "Failed to parse file entry: " + fileName, e);
-                    }
-                }
+            try {
+                int port = Integer.parseInt(portStr);
 
-                if (addr != null)
-                    addrs.add(addr);
+                addrs.add(new InetSocketAddress(denormalizeAddress(addrStr), port));
             }
+            catch (IllegalArgumentException e) {
+                U.error(log, "Failed to parse file entry: " + fileName, e);
+            }
+        }
 
         return Collections.unmodifiableCollection(addrs);
     }
@@ -274,7 +269,8 @@ private String name(InetSocketAddress addr) {
 
         SB sb = new SB();
 
-        sb.a(normalizeAddress(addr.getAddress().getHostAddress()))
+        // There is no need to normalize hostname as DNS name specification doesn't allow ':' and '_' chars.
+        sb.a(addr.isUnresolved() ? addr.getHostName() : normalizeAddress(addr.getAddress().getHostAddress()))
             .a(DELIM)
             .a(addr.getPort());
 

From 44f405a9da2a2dfc228a3302c1799c8c9ce8b5bb Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 11 Dec 2017 15:21:44 +0300
Subject: [PATCH 455/516] Backport of ignite-1267: Fixed job stealing so that
 newly joined node is able to steal jobs. (cherry picked from commit 24f9087)

---
 .../internal/GridJobExecuteRequest.java       |  53 +++++-
 .../ignite/internal/GridTaskSessionImpl.java  |  18 ++
 .../ignite/internal/IgniteComputeImpl.java    |  12 +-
 .../processors/job/GridJobProcessor.java      |  10 +-
 .../session/GridTaskSessionProcessor.java     |   6 +
 .../processors/task/GridTaskProcessor.java    |  15 +-
 .../task/GridTaskThreadContextKey.java        |   3 +
 .../processors/task/GridTaskWorker.java       |   2 +
 .../internal/GridJobStealingSelfTest.java     |   7 +-
 .../GridMultithreadedJobStealingSelfTest.java | 177 ++++++++++++++----
 10 files changed, 247 insertions(+), 56 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java
index ed431d8c2fd01..107eee3706f68 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java
@@ -23,6 +23,7 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobSibling;
 import org.apache.ignite.configuration.DeploymentMode;
@@ -31,6 +32,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
@@ -137,6 +139,13 @@ public class GridJobExecuteRequest implements Message {
     @GridDirectCollection(UUID.class)
     private Collection<UUID> top;
 
+    /** */
+    @GridDirectTransient
+    private IgnitePredicate<ClusterNode> topPred;
+
+    /** */
+    private byte[] topPredBytes;
+
     /** */
     private int[] idsOfCaches;
 
@@ -164,6 +173,8 @@ public GridJobExecuteRequest() {
      * @param startTaskTime Task execution start time.
      * @param timeout Task execution timeout.
      * @param top Topology.
+     * @param topPred Topology predicate.
+     * @param topPredBytes Marshalled topology predicate.
      * @param siblingsBytes Serialized collection of split siblings.
      * @param siblings Collection of split siblings.
      * @param sesAttrsBytes Map of session attributes.
@@ -194,6 +205,8 @@ public GridJobExecuteRequest(
             long startTaskTime,
             long timeout,
             @Nullable Collection<UUID> top,
+            @Nullable IgnitePredicate<ClusterNode> topPred,
+            byte[] topPredBytes,
             byte[] siblingsBytes,
             Collection<ComputeJobSibling> siblings,
             byte[] sesAttrsBytes,
@@ -212,7 +225,6 @@ public GridJobExecuteRequest(
             @Nullable int[] cacheIds,
             int part,
             @Nullable AffinityTopologyVersion topVer) {
-        this.top = top;
         assert sesId != null;
         assert jobId != null;
         assert taskName != null;
@@ -220,6 +232,7 @@ public GridJobExecuteRequest(
         assert job != null || jobBytes != null;
         assert sesAttrs != null || sesAttrsBytes != null || !sesFullSup;
         assert jobAttrs != null || jobAttrsBytes != null;
+        assert top != null || topPred != null || topPredBytes != null;
         assert clsLdrId != null;
         assert userVer != null;
         assert depMode != null;
@@ -234,6 +247,9 @@ public GridJobExecuteRequest(
         this.startTaskTime = startTaskTime;
         this.timeout = timeout;
         this.top = top;
+        this.topVer = topVer;
+        this.topPred = topPred;
+        this.topPredBytes = topPredBytes;
         this.siblingsBytes = siblingsBytes;
         this.siblings = siblings;
         this.sesAttrsBytes = sesAttrsBytes;
@@ -419,6 +435,21 @@ public boolean isForceLocalDeployment() {
     @Nullable public Collection<UUID> topology() {
         return top;
     }
+
+    /**
+     * @return Topology predicate.
+     */
+    public IgnitePredicate<ClusterNode> getTopologyPredicate() {
+        return topPred;
+    }
+
+    /**
+     * @return Marshalled topology predicate.
+     */
+    public byte[] getTopologyPredicateBytes() {
+        return topPredBytes;
+    }
+
     /**
      * @return {@code True} if session attributes are enabled.
      */
@@ -611,12 +642,18 @@ public AffinityTopologyVersion getTopVer() {
                 writer.incrementState();
 
             case 22:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeByteArray("topPredBytes", topPredBytes))
                     return false;
 
                 writer.incrementState();
 
             case 23:
+                if (!writer.writeMessage("topVer", topVer))
+                    return false;
+
+                writer.incrementState();
+
+            case 24:
                 if (!writer.writeString("userVer", userVer))
                     return false;
 
@@ -816,7 +853,7 @@ public AffinityTopologyVersion getTopVer() {
                 reader.incrementState();
 
             case 22:
-                topVer = reader.readMessage("topVer");
+                topPredBytes = reader.readByteArray("topPredBytes");
 
                 if (!reader.isLastRead())
                     return false;
@@ -824,6 +861,14 @@ public AffinityTopologyVersion getTopVer() {
                 reader.incrementState();
 
             case 23:
+                topVer = reader.readMessage("topVer");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 24:
                 userVer = reader.readString("userVer");
 
                 if (!reader.isLastRead())
@@ -843,7 +888,7 @@ public AffinityTopologyVersion getTopVer() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 24;
+        return 25;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionImpl.java
index dd1caa15396b3..d26925ec4ab99 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionImpl.java
@@ -27,6 +27,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJobSibling;
 import org.apache.ignite.compute.ComputeTaskSessionAttributeListener;
 import org.apache.ignite.compute.ComputeTaskSessionScope;
@@ -38,6 +39,7 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
 
@@ -108,6 +110,9 @@ public class GridTaskSessionImpl implements GridTaskSessionInternal {
     /** */
     private final Collection<UUID> top;
 
+    /** */
+    private final IgnitePredicate<ClusterNode> topPred;
+
     /** */
     private final UUID subjId;
 
@@ -121,6 +126,7 @@ public class GridTaskSessionImpl implements GridTaskSessionInternal {
      * @param taskClsName Task class name.
      * @param sesId Task session ID.
      * @param top Topology.
+     * @param topPred Topology predicate.
      * @param startTime Task execution start time.
      * @param endTime Task execution end time.
      * @param siblings Collection of siblings.
@@ -137,6 +143,7 @@ public GridTaskSessionImpl(
         String taskClsName,
         IgniteUuid sesId,
         @Nullable Collection<UUID> top,
+        @Nullable IgnitePredicate<ClusterNode> topPred,
         long startTime,
         long endTime,
         Collection<ComputeJobSibling> siblings,
@@ -154,6 +161,7 @@ public GridTaskSessionImpl(
         this.taskName = taskName;
         this.dep = dep;
         this.top = top;
+        this.topPred = topPred;
 
         // Note that class name might be null here if task was not explicitly
         // deployed.
@@ -766,8 +774,18 @@ protected boolean removeCheckpoint0(GridTaskSessionInternal ses, String key) thr
         return ctx.checkpoint().removeCheckpoint(ses, key);
     }
 
+    /**
+     * @return Topology predicate.
+     */
+    @Nullable public IgnitePredicate<ClusterNode> getTopologyPredicate() {
+        return topPred;
+    }
+
     /** {@inheritDoc} */
     @Override public Collection<UUID> getTopology() {
+        if (topPred != null)
+           return F.viewReadOnly(ctx.discovery().allNodes(), F.node2id(), topPred);
+
         return top != null ? top : F.nodeIds(ctx.discovery().allNodes());
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComputeImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComputeImpl.java
index 26c67977b1b4b..257368b7abf37 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComputeImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComputeImpl.java
@@ -50,7 +50,7 @@
 import static org.apache.ignite.internal.GridClosureCallMode.BALANCE;
 import static org.apache.ignite.internal.GridClosureCallMode.BROADCAST;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
-import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID_PREDICATE;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBJ_ID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TASK_NAME;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
@@ -261,7 +261,7 @@ public IgniteComputeImpl(GridKernalContext ctx, ClusterGroupAdapter prj, UUID su
         guard();
 
         try {
-            ctx.task().setThreadContextIfNotNull(TC_SUBGRID, prj.nodes());
+            ctx.task().setThreadContextIfNotNull(TC_SUBGRID_PREDICATE, prj.predicate());
             ctx.task().setThreadContextIfNotNull(TC_SUBJ_ID, subjId);
 
             return (R)saveOrGet(ctx.task().execute(taskName, arg));
@@ -281,7 +281,7 @@ public IgniteComputeImpl(GridKernalContext ctx, ClusterGroupAdapter prj, UUID su
         guard();
 
         try {
-            ctx.task().setThreadContextIfNotNull(TC_SUBGRID, prj.nodes());
+            ctx.task().setThreadContextIfNotNull(TC_SUBGRID_PREDICATE, prj.predicate());
             ctx.task().setThreadContextIfNotNull(TC_SUBJ_ID, subjId);
 
             return saveOrGet(ctx.task().execute(taskCls, arg));
@@ -301,7 +301,7 @@ public IgniteComputeImpl(GridKernalContext ctx, ClusterGroupAdapter prj, UUID su
         guard();
 
         try {
-            ctx.task().setThreadContextIfNotNull(TC_SUBGRID, prj.nodes());
+            ctx.task().setThreadContextIfNotNull(TC_SUBGRID_PREDICATE, prj.predicate());
             ctx.task().setThreadContextIfNotNull(TC_SUBJ_ID, subjId);
 
             return saveOrGet(ctx.task().execute(task, arg));
@@ -325,7 +325,7 @@ public <T, R> ComputeTaskInternalFuture<R> executeAsync(ComputeTask<T, R> task,
         guard();
 
         try {
-            ctx.task().setThreadContextIfNotNull(TC_SUBGRID, prj.nodes());
+            ctx.task().setThreadContextIfNotNull(TC_SUBGRID_PREDICATE, prj.predicate());
             ctx.task().setThreadContextIfNotNull(TC_SUBJ_ID, subjId);
 
             return ctx.task().execute(task, arg);
@@ -346,7 +346,7 @@ public <T, R> ComputeTaskInternalFuture<R> executeAsync(String taskName, @Nullab
         guard();
 
         try {
-            ctx.task().setThreadContextIfNotNull(TC_SUBGRID, prj.nodes());
+            ctx.task().setThreadContextIfNotNull(TC_SUBGRID_PREDICATE, prj.predicate());
             ctx.task().setThreadContextIfNotNull(TC_SUBJ_ID, subjId);
 
             return ctx.task().execute(taskName, arg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index 992e02a6fc570..02c76e2630e67 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -1044,6 +1044,13 @@ public void processJobExecuteRequest(ClusterNode node, final GridJobExecuteReque
                                     U.resolveClassLoader(dep.classLoader(), ctx.config()));
                         }
 
+                        IgnitePredicate<ClusterNode> topologyPred = req.getTopologyPredicate();
+
+                        if (topologyPred == null && req.getTopologyPredicateBytes() != null) {
+                            topologyPred = U.unmarshal(marsh, req.getTopologyPredicateBytes(),
+                                U.resolveClassLoader(dep.classLoader(), ctx.config()));
+                        }
+
                         // Note that we unmarshal session/job attributes here with proper class loader.
                         GridTaskSessionImpl taskSes = ctx.session().createTaskSession(
                             req.getSessionId(),
@@ -1052,6 +1059,7 @@ public void processJobExecuteRequest(ClusterNode node, final GridJobExecuteReque
                             dep,
                             req.getTaskClassName(),
                             req.topology(),
+                            topologyPred,
                             req.getStartTaskTime(),
                             endTime,
                             siblings,
@@ -1773,7 +1781,7 @@ private class JobEventListener implements GridJobEventListener {
                         cancelledJobs.remove(worker.getJobId(), worker);
 
                     heldJobs.remove(worker.getJobId());
-                    
+
                     if (metricsUpdateFreq > -1L)
                         updateJobMetrics();
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/session/GridTaskSessionProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/session/GridTaskSessionProcessor.java
index d660948e29baa..4e0cbb5ab2680 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/session/GridTaskSessionProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/session/GridTaskSessionProcessor.java
@@ -22,12 +22,14 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentMap;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJobSibling;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.GridTaskSessionImpl;
 import org.apache.ignite.internal.managers.deployment.GridDeployment;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
@@ -69,6 +71,7 @@ public GridTaskSessionProcessor(GridKernalContext ctx) {
      * @param dep Deployment.
      * @param taskClsName Task class name.
      * @param top Topology.
+     * @param topPred Topology predicate.
      * @param startTime Execution start time.
      * @param endTime Execution end time.
      * @param siblings Collection of siblings.
@@ -85,6 +88,7 @@ public GridTaskSessionImpl createTaskSession(
         @Nullable GridDeployment dep,
         String taskClsName,
         @Nullable Collection<UUID> top,
+        @Nullable IgnitePredicate<ClusterNode> topPred,
         long startTime,
         long endTime,
         Collection<ComputeJobSibling> siblings,
@@ -100,6 +104,7 @@ public GridTaskSessionImpl createTaskSession(
                 taskClsName,
                 sesId,
                 top,
+                topPred,
                 startTime,
                 endTime,
                 siblings,
@@ -123,6 +128,7 @@ public GridTaskSessionImpl createTaskSession(
                         taskClsName,
                         sesId,
                         top,
+                        topPred,
                         startTime,
                         endTime,
                         siblings,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index a807ad9efe5dc..04a9e6f56d417 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -69,6 +69,7 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.plugin.security.SecurityPermission;
@@ -84,6 +85,7 @@
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID_PREDICATE;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBJ_ID;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TASK_NAME;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
@@ -598,12 +600,18 @@ else if (task != null) {
         if (log.isDebugEnabled())
             log.debug("Task deployment: " + dep);
 
-        boolean fullSup = dep != null && taskCls!= null &&
+        boolean fullSup = dep != null && taskCls != null &&
             dep.annotation(taskCls, ComputeTaskSessionFullSupport.class) != null;
 
-        Collection<? extends ClusterNode> nodes = (Collection<? extends ClusterNode>)map.get(TC_SUBGRID);
+        Collection<UUID> top = null;
 
-        Collection<UUID> top = nodes != null ? F.nodeIds(nodes) : null;
+        final IgnitePredicate<ClusterNode> topPred = (IgnitePredicate<ClusterNode>)map.get(TC_SUBGRID_PREDICATE);
+
+        if (topPred == null) {
+            final Collection<ClusterNode> nodes = (Collection<ClusterNode>)map.get(TC_SUBGRID);
+
+            top = nodes != null ? F.nodeIds(nodes) : null;
+        }
 
         UUID subjId = getThreadContext(TC_SUBJ_ID);
 
@@ -625,6 +633,7 @@ else if (task != null) {
             dep,
             taskCls == null ? null : taskCls.getName(),
             top,
+            topPred,
             startTime,
             endTime,
             Collections.<ComputeJobSibling>emptyList(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
index 69dde1240585e..e798f912fdde8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskThreadContextKey.java
@@ -30,6 +30,9 @@ public enum GridTaskThreadContextKey {
     /** Projection for the task. */
     TC_SUBGRID,
 
+    /** Projection predicate for the task. */
+    TC_SUBGRID_PREDICATE,
+
     /** Timeout in milliseconds associated with the task. */
     TC_TIMEOUT,
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
index f59870b74e293..50e9ed3234f92 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
@@ -1380,6 +1380,8 @@ private void sendRequest(ComputeJobResult res) {
                         ses.getStartTime(),
                         timeout,
                         ses.getTopology(),
+                        loc ? ses.getTopologyPredicate() : null,
+                        loc ? null : U.marshal(marsh, ses.getTopologyPredicate()),
                         loc ? null : U.marshal(marsh, ses.getJobSiblings()),
                         loc ? ses.getJobSiblings() : null,
                         loc ? null : U.marshal(marsh, sesAttrs),
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java
index 21c0430751687..1e93d61d903d6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java
@@ -187,10 +187,13 @@ public void testProjectionPredicate() throws Exception {
     public void testProjectionPredicateInternalStealing() throws Exception {
         final Ignite ignite3 = startGrid(3);
 
+        final UUID node1 = ignite1.cluster().localNode().id();
+        final UUID node3 = ignite3.cluster().localNode().id();
+
         IgnitePredicate<ClusterNode> p = new P1<ClusterNode>() {
             @Override public boolean apply(ClusterNode e) {
-                return ignite1.cluster().localNode().id().equals(e.id()) ||
-                    ignite3.cluster().localNode().id().equals(e.id()); // Limit projection with only grid1 or grid3 node.
+                return node1.equals(e.id()) ||
+                    node3.equals(e.id()); // Limit projection with only grid1 or grid3 node.
             }
         };
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridMultithreadedJobStealingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridMultithreadedJobStealingSelfTest.java
index 77603c9273807..4ef97760748d4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridMultithreadedJobStealingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridMultithreadedJobStealingSelfTest.java
@@ -18,12 +18,17 @@
 package org.apache.ignite.internal;
 
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
@@ -40,6 +45,7 @@
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
+import org.eclipse.jetty.util.ConcurrentHashSet;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -50,6 +56,9 @@ public class GridMultithreadedJobStealingSelfTest extends GridCommonAbstractTest
     /** */
     private Ignite ignite;
 
+    /** */
+    private static volatile CountDownLatch jobExecutedLatch;
+
     /** */
     public GridMultithreadedJobStealingSelfTest() {
         super(false /* don't start grid*/);
@@ -77,6 +86,7 @@ public void testTwoJobsMultithreaded() throws Exception {
 
         final AtomicInteger stolen = new AtomicInteger(0);
         final AtomicInteger noneStolen = new AtomicInteger(0);
+        final ConcurrentHashSet nodes = new ConcurrentHashSet();
 
         int threadsNum = 10;
 
@@ -84,28 +94,13 @@ public void testTwoJobsMultithreaded() throws Exception {
             /** */
             @Override public void run() {
                 try {
-                    JobStealingResult res = ignite.compute().execute(JobStealingTask.class, null);
+                    JobStealingResult res = ignite.compute().execute(new JobStealingTask(2), null);
 
                     info("Task result: " + res);
 
-                    switch(res) {
-                        case NONE_STOLEN : {
-                            noneStolen.addAndGet(2);
-                            break;
-                        }
-                        case ONE_STOLEN : {
-                            noneStolen.addAndGet(1);
-                            stolen.addAndGet(1);
-                            break;
-                        }
-                        case BOTH_STOLEN: {
-                            stolen.addAndGet(2);
-                            break;
-                        }
-                        default: {
-                            assert false : "Result is: " + res;
-                        }
-                    }
+                    stolen.addAndGet(res.stolen);
+                    noneStolen.addAndGet(res.nonStolen);
+                    nodes.addAll(res.nodes);
                 }
                 catch (IgniteException e) {
                     log.error("Failed to execute task.", e);
@@ -119,20 +114,91 @@ public void testTwoJobsMultithreaded() throws Exception {
             info("Metrics [nodeId=" + g.cluster().localNode().id() +
                 ", metrics=" + g.cluster().localNode().metrics() + ']');
 
-        assert fail.get() == null : "Test failed with exception: " + fail.get();
+        assertNull("Test failed with exception: ",fail.get());
 
         // Total jobs number is threadsNum * 2
-        assert stolen.get() + noneStolen.get() == threadsNum * 2 : "Incorrect processed jobs number";
+        assertEquals("Incorrect processed jobs number",threadsNum * 2, stolen.get() + noneStolen.get());
 
-        assert stolen.get() != 0 : "No jobs were stolen.";
+        assertFalse( "No jobs were stolen.",stolen.get() == 0);
+
+        for (Ignite g : G.allGrids())
+            assertTrue("Node get no jobs.", nodes.contains(g.name()));
 
         // Under these circumstances we should not have  more than 2 jobs
         // difference.
         //(but muted to 4 due to very rare fails and low priority of fix)
-        assert Math.abs(stolen.get() - noneStolen.get()) <= 4 : "Stats [stolen=" + stolen +
-            ", noneStolen=" + noneStolen + ']';
+        assertTrue( "Stats [stolen=" + stolen + ", noneStolen=" + noneStolen + ']',
+            Math.abs(stolen.get() - noneStolen.get()) <= 4);
+    }
+
+    /**
+     * Test newly joined node can steal jobs.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testJoinedNodeCanStealJobs() throws Exception {
+        final AtomicReference<Exception> fail = new AtomicReference<>(null);
+
+        final AtomicInteger stolen = new AtomicInteger(0);
+        final AtomicInteger noneStolen = new AtomicInteger(0);
+        final ConcurrentHashSet nodes = new ConcurrentHashSet();
+
+        int threadsNum = 10;
+
+        final int jobsPerTask = 4;
+
+        jobExecutedLatch = new CountDownLatch(threadsNum);
+
+        final IgniteInternalFuture<Long> future = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+            /** */
+            @Override public void run() {
+                try {
+                    final IgniteCompute compute = ignite.compute().withAsync();
+
+                    compute.execute(new JobStealingTask(jobsPerTask), null);
+
+                    JobStealingResult res = (JobStealingResult)compute.future().get();
+
+                    info("Task result: " + res);
+
+                    stolen.addAndGet(res.stolen);
+                    noneStolen.addAndGet(res.nonStolen);
+                    nodes.addAll(res.nodes);
+                }
+                catch (IgniteException e) {
+                    log.error("Failed to execute task.", e);
+
+                    fail.getAndSet(e);
+                }
+            }
+        }, threadsNum, "JobStealingThread");
+
+        //Wait for first job begin execution.
+        jobExecutedLatch.await();
+
+        startGrid(2);
+
+        for (Ignite g : G.allGrids())
+            info("Metrics [nodeId=" + g.cluster().localNode().id() +
+                ", metrics=" + g.cluster().localNode().metrics() + ']');
+
+        future.get();
+
+        assertNull("Test failed with exception: ",fail.get());
+
+        // Total jobs number is threadsNum * 3
+        assertEquals("Incorrect processed jobs number",threadsNum * jobsPerTask, stolen.get() + noneStolen.get());
+
+        assertFalse( "No jobs were stolen.",stolen.get() == 0);
+
+        for (Ignite g : G.allGrids())
+            assertTrue("Node get no jobs.", nodes.contains(g.name()));
+
+        assertTrue( "Stats [stolen=" + stolen + ", noneStolen=" + noneStolen + ']',
+            Math.abs(stolen.get() - 2 * noneStolen.get()) <= 6);
     }
 
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -166,38 +232,50 @@ private static class JobStealingTask extends ComputeTaskAdapter<Object, JobSteal
         @LoggerResource
         private IgniteLogger log;
 
+        /** */
+        private int jobsToRun;
+
+        /** */
+        public JobStealingTask(int jobsToRun) {
+            this.jobsToRun = jobsToRun;
+        }
+
         /** {@inheritDoc} */
         @SuppressWarnings("ForLoopReplaceableByForEach")
-            @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
+        @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
             @Nullable Object arg) {
             assert subgrid.size() == 2 : "Invalid subgrid size: " + subgrid.size();
 
             Map<ComputeJobAdapter, ClusterNode> map = new HashMap<>(subgrid.size());
 
             // Put all jobs onto local node.
-            for (int i = 0; i < subgrid.size(); i++)
-                map.put(new GridJobStealingJob(2000L), ignite.cluster().localNode());
+            for (int i = 0; i < jobsToRun; i++)
+                map.put(new GridJobStealingJob(3000L), ignite.cluster().localNode());
 
             return map;
         }
 
         /** {@inheritDoc} */
         @Override public JobStealingResult reduce(List<ComputeJobResult> results) {
-            assert results.size() == 2;
+            int stolen = 0;
+            int nonStolen = 0;
+
+            Set<String> nodes = new HashSet<>(results.size());
 
-            for (ComputeJobResult res : results)
-                log.info("Job result: " + res.getData());
+            for (ComputeJobResult res : results) {
+                String data = res.getData();
 
-            Object obj0 = results.get(0).getData();
+                log.info("Job result: " + data);
 
-            if (obj0.equals(results.get(1).getData())) {
-                if (obj0.equals(ignite.name()))
-                    return JobStealingResult.NONE_STOLEN;
+                nodes.add(data);
 
-                return JobStealingResult.BOTH_STOLEN;
+                if (!data.equals(ignite.name()))
+                    stolen++;
+                else
+                    nonStolen++;
             }
 
-            return JobStealingResult.ONE_STOLEN;
+            return new JobStealingResult(stolen, nonStolen, nodes);
         }
     }
 
@@ -219,6 +297,9 @@ private static final class GridJobStealingJob extends ComputeJobAdapter {
         /** {@inheritDoc} */
         @Override public Serializable execute() {
             try {
+                if (jobExecutedLatch != null)
+                    jobExecutedLatch.countDown();
+
                 Long sleep = argument(0);
 
                 assert sleep != null;
@@ -236,14 +317,30 @@ private static final class GridJobStealingJob extends ComputeJobAdapter {
     /**
      * Job stealing result.
      */
-    private enum JobStealingResult {
+    private static class JobStealingResult {
+        /** */
+        int stolen;
+
         /** */
-        BOTH_STOLEN,
+        int nonStolen;
 
         /** */
-        ONE_STOLEN,
+        Set nodes;
 
         /** */
-        NONE_STOLEN
+        public JobStealingResult(int stolen, int nonStolen, Set nodes) {
+            this.stolen = stolen;
+            this.nonStolen = nonStolen;
+            this.nodes = nodes;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "JobStealingResult{" +
+                "stolen=" + stolen +
+                ", nonStolen=" + nonStolen +
+                ", nodes=" + Arrays.toString(nodes.toArray()) +
+                '}';
+        }
     }
 }
\ No newline at end of file

From e0c1052c2f2c54bd7f99d3dbd3505c436d10d0c1 Mon Sep 17 00:00:00 2001
From: xmitya <dkarachentsev@gridgain.com>
Date: Mon, 11 Dec 2017 14:11:40 +0300
Subject: [PATCH 456/516] IGNITE-7159 - Fix
 IgniteCacheAtomicExpiryPolicyWithStoreTest.testGetReadThrough

(cherry picked from commit e3d70a8)
---
 .../ignite/internal/processors/cache/GridCacheUtils.java    | 6 ++++--
 .../cache/distributed/dht/GridPartitionedGetFuture.java     | 2 +-
 .../distributed/dht/GridPartitionedSingleGetFuture.java     | 3 ++-
 .../cache/distributed/near/GridNearGetFuture.java           | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 171be1e362918..1c60cdbcdc5b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1743,6 +1743,7 @@ public static TransactionConfiguration transactionConfiguration(final @Nullable
      * @param log Logger.
      * @param cctx Cache context.
      * @param key Key.
+     * @param expiryPlc Expiry policy.
      * @param readThrough Read through.
      * @param skipVals Skip values.
      */
@@ -1751,6 +1752,7 @@ public static TransactionConfiguration transactionConfiguration(final @Nullable
         final IgniteLogger log,
         final GridCacheContext cctx,
         final @Nullable KeyCacheObject key,
+        final @Nullable IgniteCacheExpiryPolicy expiryPlc,
         boolean readThrough,
         boolean skipVals
     ) {
@@ -1769,8 +1771,8 @@ private void process(KeyCacheObject key, CacheObject val, GridCacheVersion ver,
                         entry.initialValue(
                             val,
                             ver,
-                            0,
-                            0,
+                            expiryPlc == null ? 0 : expiryPlc.forCreate(),
+                            expiryPlc == null ? 0 : toExpireTime(expiryPlc.forCreate()),
                             false,
                             topVer,
                             GridDrType.DR_BACKUP,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 560bf185f062a..aee6df7762c62 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -350,7 +350,7 @@ private void map(
             }
             else {
                 MiniFuture fut = new MiniFuture(n, mappedKeys, topVer,
-                    CU.createBackupPostProcessingClosure(topVer, log, cctx, null, readThrough, skipVals));
+                    CU.createBackupPostProcessingClosure(topVer, log, cctx, null, expiryPlc, readThrough, skipVals));
 
                 GridCacheMessage req = new GridNearGetRequest(
                     cctx.cacheId(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index cda7d5201e596..dc0f97bb4de0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -293,7 +293,8 @@ private void map(AffinityTopologyVersion topVer) {
 
             boolean needVer = this.needVer;
 
-            final BackupPostProcessingClosure postClos = CU.createBackupPostProcessingClosure(topVer, log, cctx, key, readThrough, skipVals);
+            final BackupPostProcessingClosure postClos = CU.createBackupPostProcessingClosure(topVer, log,
+                cctx, key, expiryPlc, readThrough, skipVals);
 
             if (postClos != null) {
                 // Need version to correctly store value.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index 95ad6d82222c8..d5ed30cc133e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -380,7 +380,7 @@ private void map(
                 }
 
                 MiniFuture fut = new MiniFuture(n, mappedKeys, saved, topVer,
-                    CU.createBackupPostProcessingClosure(topVer, log, cctx, null, readThrough, skipVals));
+                    CU.createBackupPostProcessingClosure(topVer, log, cctx, null, expiryPlc, readThrough, skipVals));
 
                 GridCacheMessage req = new GridNearGetRequest(
                     cctx.cacheId(),

From 69cf8cc3abc8b8fe61787e1241702b5e2a01f4ca Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 18 Dec 2017 10:47:08 +0300
Subject: [PATCH 457/516] Fail overtimed test.

---
 .../cache/distributed/IgniteCacheMessageWriteTimeoutTest.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index da9f11f6a7dd8..46055251cc469 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -75,6 +75,8 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testMessageQueueLimit() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-13139");
+
         for (int i = 0; i < 15; i++) {
             log.info("Iteration: " + i);
 

From a9167d6d5f11426199e99aa2dcd03e79a84d53a9 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Wed, 20 Dec 2017 13:25:15 +0300
Subject: [PATCH 458/516] IGNITE-7197 Avoid NPE in services() by waiting on
 latch

cherry-picked from 226711357db742a5073a98ef179997480e1b1a56
---
 .../service/GridServiceProcessor.java         |  55 ++++--
 .../processors/task/GridTaskProcessor.java    |   2 +
 .../internal/GridJobServicesAddNodeTest.java  | 178 ++++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |   2 +
 4 files changed, 225 insertions(+), 12 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/GridJobServicesAddNodeTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index b0b9182c48dff..8914094be3d54 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -32,6 +32,7 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
@@ -172,11 +173,14 @@ public class GridServiceProcessor extends GridProcessorAdapter {
     private ThreadLocal<String> svcName = new ThreadLocal<>();
 
     /** Service cache. */
-    private IgniteInternalCache<Object, Object> cache;
+    private volatile IgniteInternalCache<Object, Object> serviceCache;
 
     /** Topology listener. */
     private final DiscoveryEventListener topLsnr = new TopologyListener();
 
+    /** */
+    private final CountDownLatch startLatch = new CountDownLatch(1);
+
     static {
         Set<IgniteProductVersion> versions = new TreeSet<>(new Comparator<IgniteProductVersion>() {
             @Override public int compare(final IgniteProductVersion o1, final IgniteProductVersion o2) {
@@ -256,19 +260,20 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
         if (ctx.isDaemon())
             return;
 
-        cache = ctx.cache().utilityCache();
-
         if (!ctx.clientNode())
             ctx.event().addDiscoveryEventListener(topLsnr, EVTS);
 
+        updateUtilityCache();
+        startLatch.countDown();
+
         try {
             if (ctx.deploy().enabled())
                 ctx.cache().context().deploy().ignoreOwnership(true);
 
             if (!ctx.clientNode()) {
-                assert cache.context().affinityNode();
+                assert serviceCache.context().affinityNode();
 
-                cache.context().continuousQueries().executeInternalQuery(new ServiceEntriesListener(),
+                serviceCache.context().continuousQueries().executeInternalQuery(new ServiceEntriesListener(),
                     null,
                     true,
                     true,
@@ -281,7 +286,7 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
                     @Override public void run() {
                         try {
                             Iterable<CacheEntryEvent<?, ?>> entries =
-                                cache.context().continuousQueries().existingEntries(false, null);
+                                serviceCache.context().continuousQueries().existingEntries(false, null);
 
                             onSystemCacheUpdated(entries);
                         }
@@ -307,6 +312,23 @@ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteChe
             log.debug("Started service processor.");
     }
 
+    /*
+     *
+     */
+    public void updateUtilityCache() {
+        serviceCache = ctx.cache().utilityCache();
+    }
+
+    /**
+     * @return Service cache.
+     */
+    private IgniteInternalCache<Object, Object> serviceCache() {
+        if (serviceCache == null)
+            U.awaitQuiet(startLatch);
+
+        return serviceCache;
+    }
+
     /** {@inheritDoc} */
     @Override public void onKernalStop(boolean cancel) {
         if (ctx.isDaemon())
@@ -624,7 +646,7 @@ private IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs,
                     if (cfgsCp.size() == 1)
                         writeServiceToCache(res, cfgsCp.get(0));
                     else if (cfgsCp.size() > 1) {
-                        try (Transaction tx = cache.txStart(PESSIMISTIC, READ_COMMITTED)) {
+                        try (Transaction tx = serviceCache().txStart(PESSIMISTIC, READ_COMMITTED)) {
                             for (ServiceConfiguration cfg : cfgsCp) {
                                 try {
                                     writeServiceToCache(res, cfg);
@@ -720,7 +742,7 @@ private void writeServiceToCache(GridServiceDeploymentCompoundFuture res, Servic
 
             GridServiceDeploymentKey key = new GridServiceDeploymentKey(name);
 
-            GridServiceDeployment dep = (GridServiceDeployment)cache.getAndPutIfAbsent(key,
+            GridServiceDeployment dep = (GridServiceDeployment)serviceCache().getAndPutIfAbsent(key,
                 new GridServiceDeployment(ctx.localNodeId(), cfg));
 
             if (dep != null) {
@@ -844,7 +866,7 @@ public IgniteInternalFuture<?> cancelAll(Collection<String> svcNames) {
 
             List<String> toRollback = new ArrayList<>();
 
-            try (Transaction tx = cache.txStart(PESSIMISTIC, READ_COMMITTED)) {
+            try (Transaction tx = serviceCache().txStart(PESSIMISTIC, READ_COMMITTED)) {
                 for (String name : svcNames) {
                     if (res == null)
                         res = new GridCompoundFuture<>();
@@ -917,7 +939,7 @@ private CancelResult removeServiceFromCache(String name) throws IgniteCheckedExc
             GridServiceDeploymentKey key = new GridServiceDeploymentKey(name);
 
             try {
-                if (cache.getAndRemove(key) == null) {
+                if (serviceCache().getAndRemove(key) == null) {
                     // Remove future from local map if service was not deployed.
                     undepFuts.remove(name, fut);
 
@@ -944,6 +966,8 @@ private CancelResult removeServiceFromCache(String name) throws IgniteCheckedExc
      * @return Service topology.
      */
     public Map<UUID, Integer> serviceTopology(String name, long timeout) throws IgniteCheckedException {
+        IgniteInternalCache<Object, Object> cache = serviceCache();
+
         ClusterNode node = cache.affinity().mapKeyToNode(name);
 
         if (node.version().compareTo(ServiceTopologyCallable.SINCE_VER) >= 0) {
@@ -995,7 +1019,7 @@ public Collection<ServiceDescriptor> serviceDescriptors() {
             ServiceDescriptorImpl desc = new ServiceDescriptorImpl(dep);
 
             try {
-                GridServiceAssignments assigns = (GridServiceAssignments)cache.getForcePrimary(
+                GridServiceAssignments assigns = (GridServiceAssignments)serviceCache().getForcePrimary(
                     new GridServiceAssignmentsKey(dep.configuration().getName()));
 
                 if (assigns != null) {
@@ -1160,8 +1184,9 @@ public <T> Collection<T> services(String name) {
      * @throws IgniteCheckedException If failed.
      */
     private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer) throws IgniteCheckedException {
-        ServiceConfiguration cfg = dep.configuration();
+        IgniteInternalCache<Object, Object> cache = serviceCache();
 
+        ServiceConfiguration cfg = dep.configuration();
         Object nodeFilter = cfg.getNodeFilter();
 
         if (nodeFilter != null)
@@ -1538,6 +1563,8 @@ private void cancel(Iterable<ServiceContextImpl> ctxs, int cancelCnt) {
     @SuppressWarnings("unchecked")
     private Iterator<Cache.Entry<Object, Object>> serviceEntries(IgniteBiPredicate<Object, Object> p) {
         try {
+            IgniteInternalCache<Object, Object> cache = serviceCache();
+
             if (!cache.context().affinityNode()) {
                 ClusterNode oldestSrvNode =
                     ctx.discovery().oldestAliveCacheServerNode(AffinityTopologyVersion.NONE);
@@ -1761,6 +1788,8 @@ private void processDeployment(CacheEntryEvent<GridServiceDeploymentKey, GridSer
             GridServiceAssignmentsKey key = new GridServiceAssignmentsKey(name);
 
             // Remove assignment on primary node in case of undeploy.
+            IgniteInternalCache<Object, Object> cache = serviceCache();
+
             if (cache.cache().affinity().isPrimary(ctx.discovery().localNode(), key)) {
                 try {
                     cache.getAndRemove(key);
@@ -1937,6 +1966,8 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                         }
 
                         // Clean up zombie assignments.
+                        IgniteInternalCache<Object, Object> cache = serviceCache();
+
                         for (Cache.Entry<Object, Object> e :
                             cache.entrySetx(CU.cachePrimary(ctx.grid().affinity(cache.name()), ctx.grid().localNode()))) {
                             if (!(e.getKey() instanceof GridServiceAssignmentsKey))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 04a9e6f56d417..995a25618a89b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -194,6 +194,8 @@ private IgniteClientDisconnectedCheckedException disconnectedError(@Nullable Ign
             lock.writeUnlock();
         }
 
+        startLatch.countDown();
+
         int size = tasks.size();
 
         if (size > 0) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridJobServicesAddNodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridJobServicesAddNodeTest.java
new file mode 100644
index 0000000000000..8c97ef4f19b89
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridJobServicesAddNodeTest.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.service.DummyService;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.typedef.CAX;
+import org.apache.ignite.internal.util.typedef.CIX1;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+
+/**
+ * Tests multiple parallel jobs execution, accessing services(), while starting new nodes.
+ */
+@GridCommonTest(group = "Kernal Self")
+public class GridJobServicesAddNodeTest extends GridCommonAbstractTest {
+    /** */
+    private static final int LOG_MOD = 100;
+
+    /** */
+    private static final int MAX_ADD_NODES = 64;
+
+    /** IP finder. */
+    private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrid(1);
+        startGrid(2);
+
+        assertEquals(2, grid(1).cluster().nodes().size());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(ipFinder);
+
+        c.setDiscoverySpi(disco);
+
+        TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
+
+        commSpi.setSharedMemoryPort(-1);
+
+        c.setCommunicationSpi(commSpi);
+
+        return c;
+    }
+
+    /**
+     * @throws Exception If test failed.
+     */
+    public void testServiceDescriptorsJob() throws Exception {
+        final int tasks = 5000;
+        final int threads = 10;
+
+        final Ignite ignite1 = grid(1);
+        final CountDownLatch latch = new CountDownLatch(tasks);
+        final AtomicInteger jobsCnt = new AtomicInteger();
+        final AtomicInteger resCnt = new AtomicInteger();
+
+        ignite1.services().deployClusterSingleton("jobsSvc", new DummyService());
+
+        GridTestUtils.runMultiThreadedAsync(new CAX() {
+            @Override public void applyx() throws IgniteCheckedException {
+                while (true) {
+                    int cnt = jobsCnt.incrementAndGet();
+
+                    if (cnt > 5000)
+                        break;
+
+                    IgniteCallable<Boolean> job;
+
+                    job = new ServiceDescriptorsJob();
+
+                    IgniteCompute comp = ignite1.compute().withAsync();
+
+                    comp.call(job);
+
+                    ComputeTaskFuture<Boolean> fut = comp.future();
+
+                    if (cnt % LOG_MOD == 0)
+                        X.println("Submitted jobs: " + cnt);
+
+                    fut.listen(new CIX1<IgniteFuture<Boolean>>() {
+                        @Override public void applyx(IgniteFuture<Boolean> f) {
+                            try {
+                                assert f.get();
+
+                                long cnt = resCnt.incrementAndGet();
+
+                                if (cnt % LOG_MOD == 0)
+                                    X.println("Results count: " + cnt);
+                            }
+                            finally {
+                                latch.countDown();
+                            }
+                        }
+                    });
+
+                    IgniteUtils.sleep(5);
+                }
+            }
+        }, threads, "TEST-THREAD");
+
+        int additionalNodesStarted = 0;
+        while (!latch.await(threads, TimeUnit.MILLISECONDS)) {
+            if (additionalNodesStarted++ <= MAX_ADD_NODES) {
+                startGrid(2 + additionalNodesStarted);
+            }
+        }
+
+        assertEquals("Jobs cnt != Results cnt", jobsCnt.get() - threads, resCnt.get());
+    }
+
+    /**
+     * Test service enumerating job.
+     */
+    @SuppressWarnings({"PublicInnerClass"})
+    public static class ServiceDescriptorsJob implements IgniteCallable<Boolean> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public Boolean call() throws Exception {
+            try {
+                return ignite.services().serviceDescriptors().iterator().hasNext();
+            } catch (Exception e) {
+                e.printStackTrace();
+
+                return false;
+            } finally {
+                Thread.sleep(10);
+            }
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 7e299ca9a8cff..390a90c95756e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.GridJobCollisionCancelSelfTest;
 import org.apache.ignite.internal.GridJobContextSelfTest;
 import org.apache.ignite.internal.GridJobMasterLeaveAwareSelfTest;
+import org.apache.ignite.internal.GridJobServicesAddNodeTest;
 import org.apache.ignite.internal.GridJobStealingSelfTest;
 import org.apache.ignite.internal.GridJobStealingZeroActiveJobsSelfTest;
 import org.apache.ignite.internal.GridJobSubjectIdSelfTest;
@@ -158,6 +159,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(PublicThreadpoolStarvationTest.class);
         suite.addTestSuite(IgniteComputeJobOneThreadTest.class);
         suite.addTestSuite(StripedExecutorTest.class);
+        suite.addTestSuite(GridJobServicesAddNodeTest.class);
 
         return suite;
     }

From b8e7d578a67b7821a28077ef4c5ee54dcdf3e161 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Wed, 20 Dec 2017 16:40:55 +0300
Subject: [PATCH 459/516] IGNITE-7206 Stop pings if current node is stopping.

(cherry picked from commit db78735)
---
 .../org/apache/ignite/spi/discovery/tcp/ServerImpl.java    | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index a89eb08b2e2f8..e06aa8cc15e44 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -720,6 +720,13 @@ private boolean pingNode(TcpDiscoveryNode node) {
                             break;
                         else if (!spi.failureDetectionTimeoutEnabled() && reconCnt == spi.getReconnectCount())
                             break;
+
+                        if (spi.isNodeStopping0()) {
+                            if (log.isDebugEnabled())
+                                log.debug("Stop pinging node, because node is stopping: [rmtNodeId=" + nodeId + ']');
+
+                            break;
+                        }
                     }
                     finally {
                         U.closeQuiet(sock);

From b6f1ab7a4cc3be5a09d14e4775a0f45ac09c87a5 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 15 Dec 2017 15:24:44 +0300
Subject: [PATCH 460/516] GG-13213 - Fix. Client node may never stop.

---
 .../ignite/internal/util/IgniteUtils.java     | 20 ++++++++++++++++---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  5 ++++-
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 6fc4c261b8e52..4d57212f6af39 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -4574,11 +4574,24 @@ public static void interrupt(Iterable<? extends Thread> workers) {
      * @return {@code true} if thread has finished, {@code false} otherwise.
      */
     public static boolean join(@Nullable Thread t, @Nullable IgniteLogger log) {
-        if (t != null)
+        return join(t, log, 0);
+    }
+
+    /**
+     * Waits for completion of a given thread. If thread is {@code null} then
+     * this method returns immediately returning {@code true}
+     *
+     * @param t Thread to join.
+     * @param log Logger for logging errors.
+     * @param timeout Join timeout.
+     * @return {@code true} if thread has finished, {@code false} otherwise.
+     */
+    public static boolean join(@Nullable Thread t, @Nullable IgniteLogger log, long timeout) {
+        if (t != null) {
             try {
-                t.join();
+                t.join(timeout);
 
-                return true;
+                return !t.isAlive();
             }
             catch (InterruptedException ignore) {
                 warn(log, "Got interrupted while waiting for completion of a thread: " + t);
@@ -4587,6 +4600,7 @@ public static boolean join(@Nullable Thread t, @Nullable IgniteLogger log) {
 
                 return false;
             }
+        }
 
         return true;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index c7c2ffa73a1e4..2651e93351fce 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -306,7 +306,10 @@ class ClientImpl extends TcpDiscoveryImpl {
 
         U.join(msgWorker, log);
         U.join(sockWriter, log);
-        U.join(sockReader, log);
+
+        // TODO Need to understand why SocketReader is not got interrupted.
+        while (!U.join(sockReader, log, 200))
+            U.interrupt(sockReader);
 
         timer.cancel();
 

From abfb419db6755a0b72474261954df2479e0340a4 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Wed, 20 Dec 2017 15:09:36 +0300
Subject: [PATCH 461/516] ignite-gg-13232 ScanQuery transformer applies to all
 result pages [ignite-5804]

---
 .../cache/query/GridCacheQueryManager.java    | 48 +++++++++--
 .../GridCacheQueryTransformerSelfTest.java    | 79 +++++++++++++++++++
 2 files changed, 119 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 47f1bed98680c..89e70b4e5f0fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -643,7 +643,8 @@ private Object unwrapIfNeeded(CacheObject obj, CacheObjectContext coctx) {
      */
     @SuppressWarnings("unchecked")
     private QueryResult<K, V> executeQuery(GridCacheQueryAdapter<?> qry,
-        @Nullable Object[] args, boolean loc, @Nullable UUID subjId, @Nullable String taskName, Object rcpt)
+        @Nullable Object[] args, @Nullable IgniteClosure<Cache.Entry<K, V>, Object> transformer,
+        boolean loc, @Nullable UUID subjId, @Nullable String taskName, Object rcpt)
         throws IgniteCheckedException {
         if (qry.type() == null) {
             assert !loc;
@@ -671,7 +672,7 @@ private QueryResult<K, V> executeQuery(GridCacheQueryAdapter<?> qry,
                 resKey = null;
         }
         else
-            res = new QueryResult<>(qry.type(), rcpt);
+            res = new QueryResult<>(qry.type(), rcpt, transformer);
 
         GridCloseableIterator<IgniteBiTuple<K, V>> iter;
 
@@ -1533,7 +1534,7 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
                 GridCacheQueryType type;
 
                 res = loc ?
-                    executeQuery(qry, qryInfo.arguments(), loc, qry.subjectId(), taskName,
+                    executeQuery(qry, qryInfo.arguments(), trans, loc, qry.subjectId(), taskName,
                         recipient(qryInfo.senderId(), qryInfo.requestId())) :
                     queryResult(qryInfo, taskName);
 
@@ -1542,6 +1543,7 @@ protected void runQuery(GridCacheQueryInfo qryInfo) {
 
                 iter = res.iterator(recipient(qryInfo.senderId(), qryInfo.requestId()));
                 type = res.type();
+                trans = res.transformer();
 
                 final GridCacheAdapter<K, V> cache = cctx.cache();
 
@@ -1833,6 +1835,11 @@ protected GridCloseableIterator scanQueryLocal(final GridCacheQueryAdapter qry,
 
             final boolean readEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ);
 
+            final IgniteClosure<Cache.Entry<K, V>, Object> transformer =
+                (IgniteClosure<Cache.Entry<K, V>, Object>)qry.transform();
+
+            injectResources(transformer);
+
             return new GridCloseableIteratorAdapter<Object>() {
                 @Override protected Object onNext() throws IgniteCheckedException {
                     long start = statsEnabled ? System.nanoTime() : 0L;
@@ -1867,9 +1874,7 @@ protected GridCloseableIterator scanQueryLocal(final GridCacheQueryAdapter qry,
                             null));
                     }
 
-                    IgniteClosure transform = qry.transform();
-
-                    if (transform == null)
+                    if (transformer == null)
                         return next;
 
                     Cache.Entry<K, V> entry;
@@ -1879,7 +1884,7 @@ protected GridCloseableIterator scanQueryLocal(final GridCacheQueryAdapter qry,
                     else
                         entry = cctx.cache().getEntry(next.getKey());
 
-                    return transform.apply(entry);
+                    return transformer.apply(entry);
                 }
 
                 @Override protected boolean onHasNext() throws IgniteCheckedException {
@@ -1964,7 +1969,8 @@ protected GridCloseableIterator scanQueryLocal(final GridCacheQueryAdapter qry,
 
         if (exec) {
             try {
-                fut.onDone(executeQuery(qryInfo.query(), qryInfo.arguments(), false,
+                fut.onDone(executeQuery(qryInfo.query(), qryInfo.arguments(),
+                    (IgniteClosure<Cache.Entry<K, V>, Object>) qryInfo.transformer(),false,
                     qryInfo.query().subjectId(), taskName, recipient(qryInfo.senderId(), qryInfo.requestId())));
             }
             catch (Throwable e) {
@@ -2749,6 +2755,9 @@ private static class QueryResult<K, V> extends CachedResult<IgniteBiTuple<K, V>>
         /** */
         private final GridCacheQueryType type;
 
+        /** */
+        private final IgniteClosure<Cache.Entry<K, V>, Object> transformer;
+
         /**
          * @param type Query type.
          * @param rcpt ID of the recipient.
@@ -2757,6 +2766,21 @@ private QueryResult(GridCacheQueryType type, Object rcpt) {
             super(rcpt);
 
             this.type = type;
+
+            this.transformer = null;
+        }
+
+        /**
+         * @param type Query type.
+         * @param rcpt ID of the recipient.
+         * @param transformer Transformer.
+         */
+        private QueryResult(GridCacheQueryType type, Object rcpt, IgniteClosure<Cache.Entry<K, V>, Object> transformer) {
+            super(rcpt);
+
+            this.type = type;
+
+            this.transformer = transformer;
         }
 
         /**
@@ -2765,6 +2789,14 @@ private QueryResult(GridCacheQueryType type, Object rcpt) {
         public GridCacheQueryType type() {
             return type;
         }
+
+        /**
+         * Returns transformer.
+         * @return Transformer.
+         */
+        public IgniteClosure<Cache.Entry<K, V>, Object> transformer() {
+            return transformer;
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryTransformerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryTransformerSelfTest.java
index e7e173bf5c4fc..30863fc60fe8f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryTransformerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryTransformerSelfTest.java
@@ -553,6 +553,85 @@ public void testUnsupported() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPageSize() throws Exception {
+        IgniteCache<Integer, Value> cache = grid().createCache("test-cache");
+
+        int numEntries = 10_000;
+        int pageSize = 3;
+
+        try {
+            for (int i = 0; i < numEntries; i++)
+                cache.put(i, new Value("str" + i, i));
+
+            IgniteClosure<Cache.Entry<Integer, Value>, Integer> transformer =
+                new IgniteClosure<Cache.Entry<Integer, Value>, Integer>() {
+                    @Override public Integer apply(Cache.Entry<Integer, Value> e) {
+                        return e.getValue().idx;
+                    }
+                };
+
+            ScanQuery<Integer, Value> query = new ScanQuery<>();
+            query.setPageSize(pageSize);
+
+            List<Integer> res = cache.query(query, transformer).getAll();
+
+            assertEquals(numEntries, res.size());
+
+            Collections.sort(res);
+
+            for (int i = 0; i < numEntries; i++)
+                assertEquals(i, res.get(i).intValue());
+        }
+        finally {
+            cache.destroy();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLocalInjection() throws Exception {
+        IgniteCache<Integer, Value> cache = grid().createCache("test-cache");
+
+        try {
+            for (int i = 0; i < 50; i++)
+                cache.put(i, new Value("str" + i, i * 100));
+
+            Collection<List<Boolean>> lists = grid().compute().broadcast(new IgniteCallable<List<Boolean>>() {
+                @IgniteInstanceResource
+                private Ignite ignite;
+
+                @Override public List<Boolean> call() throws Exception {
+                    IgniteClosure<Cache.Entry<Integer, Value>, Boolean> transformer =
+                        new IgniteClosure<Cache.Entry<Integer, Value>, Boolean>() {
+                            @IgniteInstanceResource
+                            Ignite ignite;
+
+                            @Override public Boolean apply(Cache.Entry<Integer, Value> e) {
+                                return ignite != null;
+                            }
+                        };
+
+                    return ignite.cache("test-cache").query(new ScanQuery<Integer, Value>().setLocal(true),
+                        transformer).getAll();
+                }
+            });
+
+            List<Boolean> res = new ArrayList<>(F.flatCollections(lists));
+
+            assertEquals(50, res.size());
+
+            for (int i = 0; i < 50; i++)
+                assertEquals(Boolean.TRUE, res.get(i));
+        }
+        finally {
+            cache.destroy();
+        }
+    }
+
     /**
      */
     private static class Value {

From 1b6baecb35251382ac9658acc7a234a876132b70 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 21 Dec 2017 17:03:37 +0300
Subject: [PATCH 462/516] IGNITE-7181 - Fix IGNITE-7086 doesn't work with
 PESSIMISTIC REPEATABLE_READ transaction

(cherry picked from commit e53605d)
---
 .../processors/cache/GridCacheUtils.java      | 21 +++++++--
 .../dht/GridPartitionedGetFuture.java         |  2 +-
 .../dht/GridPartitionedSingleGetFuture.java   |  2 +-
 .../colocated/GridDhtColocatedLockFuture.java |  6 +++
 .../distributed/near/GridNearGetFuture.java   |  2 +-
 .../distributed/near/GridNearLockFuture.java  |  7 +++
 .../store/CacheStoreReadFromBackupTest.java   | 43 +++++++++++++----
 ...eTransactionalStoreReadFromBackupTest.java | 46 +++++++++++++++++++
 8 files changed, 115 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 1c60cdbcdc5b3..a78a9226b065a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1753,6 +1753,7 @@ public static TransactionConfiguration transactionConfiguration(final @Nullable
         final GridCacheContext cctx,
         final @Nullable KeyCacheObject key,
         final @Nullable IgniteCacheExpiryPolicy expiryPlc,
+        final long ttl,
         boolean readThrough,
         boolean skipVals
     ) {
@@ -1768,11 +1769,25 @@ private void process(KeyCacheObject key, CacheObject val, GridCacheVersion ver,
                     try {
                         entry = colocated.entryEx(key, topVer);
 
+                        long ttl0 = 0;
+                        long expTime = 0;
+
+                        if (expiryPlc != null) {
+                            ttl0 = expiryPlc.forCreate();
+
+                            expTime = toExpireTime(ttl0);
+                        }
+                        else if (ttl > 0) {
+                            ttl0 = ttl;
+
+                            expTime = toExpireTime(ttl0);
+                        }
+
                         entry.initialValue(
                             val,
                             ver,
-                            expiryPlc == null ? 0 : expiryPlc.forCreate(),
-                            expiryPlc == null ? 0 : toExpireTime(expiryPlc.forCreate()),
+                            ttl0,
+                            expTime,
                             false,
                             topVer,
                             GridDrType.DR_BACKUP,
@@ -1799,7 +1814,7 @@ private void process(KeyCacheObject key, CacheObject val, GridCacheVersion ver,
             }
 
             @Override public void apply(CacheObject val, GridCacheVersion ver) {
-                process(key, val, ver, cctx.dht());
+                process(key, val, ver, cctx.cache().isNear() ? ((GridNearCacheAdapter)cctx.cache()).dht() : cctx.dht());
             }
 
             @Override public void apply(Collection<GridCacheEntryInfo> infos) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index aee6df7762c62..f724f51e5dd7f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -350,7 +350,7 @@ private void map(
             }
             else {
                 MiniFuture fut = new MiniFuture(n, mappedKeys, topVer,
-                    CU.createBackupPostProcessingClosure(topVer, log, cctx, null, expiryPlc, readThrough, skipVals));
+                    CU.createBackupPostProcessingClosure(topVer, log, cctx, null, expiryPlc, 0, readThrough, skipVals));
 
                 GridCacheMessage req = new GridNearGetRequest(
                     cctx.cacheId(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index dc0f97bb4de0c..51a2d9594019a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -294,7 +294,7 @@ private void map(AffinityTopologyVersion topVer) {
             boolean needVer = this.needVer;
 
             final BackupPostProcessingClosure postClos = CU.createBackupPostProcessingClosure(topVer, log,
-                cctx, key, expiryPlc, readThrough, skipVals);
+                cctx, key, expiryPlc, 0, readThrough, skipVals);
 
             if (postClos != null) {
                 // Need version to correctly store value.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index 4c37c538d4950..39ce4287fe700 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -1561,6 +1561,12 @@ void onResult(GridNearLockResponse res) {
                         // Set value to detached entry.
                         entry.resetFromPrimary(newVal, dhtVer);
 
+                        CU.BackupPostProcessingClosure clos = CU.createBackupPostProcessingClosure(topVer,
+                            log, cctx, k, null, createTtl, cctx.readThrough(), !retval);
+
+                        if (clos != null)
+                            clos.apply(newVal, res.dhtVersion(i));
+
                         tx.hasRemoteLocks(true);
 
                         if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index d5ed30cc133e7..0ca9b23a7d180 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -380,7 +380,7 @@ private void map(
                 }
 
                 MiniFuture fut = new MiniFuture(n, mappedKeys, saved, topVer,
-                    CU.createBackupPostProcessingClosure(topVer, log, cctx, null, expiryPlc, readThrough, skipVals));
+                    CU.createBackupPostProcessingClosure(topVer, log, cctx, null, expiryPlc, 0, readThrough, skipVals));
 
                 GridCacheMessage req = new GridNearGetRequest(
                     cctx.cacheId(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index d1f3fac1b5416..1f5e89182cd4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -1666,6 +1666,13 @@ void onResult(GridNearLockResponse res) {
                             entry.resetFromPrimary(newVal, lockVer, dhtVer, node.id(), topVer);
 
                             if (inTx()) {
+                                CU.BackupPostProcessingClosure clos =
+                                    CU.createBackupPostProcessingClosure(topVer, log, cctx, k,
+                                        null, createTtl, cctx.readThrough(), !retval);
+
+                                if (clos != null)
+                                    clos.apply(newVal, res.dhtVersion(i));
+
                                 tx.hasRemoteLocks(true);
 
                                 if (implicitTx() && tx.onePhaseCommit()) {
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreReadFromBackupTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreReadFromBackupTest.java
index d8913dcf1992f..5c52625aebddb 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreReadFromBackupTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreReadFromBackupTest.java
@@ -32,7 +32,11 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.lang.IgniteBiInClosure;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
@@ -82,7 +86,11 @@ private CacheConfiguration cacheConfig(String cacheName) {
 
     /** */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        return super.getConfiguration(gridName).setCacheConfiguration(cacheConfig(CACHE_NAME));
+        IgniteConfiguration cfg = super.getConfiguration(gridName).setCacheConfiguration(cacheConfig(CACHE_NAME));
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(LOCAL_IP_FINDER);
+
+        return cfg;
     }
 
     /** {@inheritDoc} */
@@ -105,7 +113,7 @@ public void testReplicated() throws Exception {
         backups = 0;
         near = false;
 
-        checkReadFromBackup();
+        startAndCheckReadFromBackup();
     }
 
     /**
@@ -116,7 +124,7 @@ public void testPartitioned() throws Exception {
         backups = 1;
         near = false;
 
-        checkReadFromBackup();
+        startAndCheckReadFromBackup();
     }
 
     /**
@@ -127,7 +135,7 @@ public void testNearReplicated() throws Exception {
         backups = 0;
         near = true;
 
-        checkReadFromBackup();
+        startAndCheckReadFromBackup();
     }
 
     /**
@@ -138,17 +146,36 @@ public void testNearPartitioned() throws Exception {
         backups = 1;
         near = true;
 
-        checkReadFromBackup();
+        startAndCheckReadFromBackup();
     }
 
     /**
      * @throws Exception If failed.
      */
-    private void checkReadFromBackup() throws Exception {
+    private void startAndCheckReadFromBackup() throws Exception {
         startGridsMultiThreaded(2, true);
 
-        checkReadSingleFromBackup();
-        checkReadAllFromBackup();
+        checkReadFromBackup(null, null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    void checkReadFromBackup(TransactionConcurrency txConcurrency, TransactionIsolation txIsolation) throws Exception {
+        Transaction tx = txConcurrency != null && txIsolation != null
+            ? grid(0).transactions().txStart(txConcurrency, txIsolation)
+            : null;
+
+        try {
+            checkReadSingleFromBackup();
+            checkReadAllFromBackup();
+        }
+        finally {
+            if (tx != null)
+                tx.close();
+
+            grid(0).cache(CACHE_NAME).clear();
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java
index 4837936621f46..e02544056976f 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java
@@ -20,6 +20,11 @@
 import org.apache.ignite.cache.CacheAtomicityMode;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
 
 /**
  *
@@ -29,4 +34,45 @@ public class CacheTransactionalStoreReadFromBackupTest extends CacheStoreReadFro
     @Override protected CacheAtomicityMode atomicityMode() {
         return TRANSACTIONAL;
     }
+
+    /** {@inheritDoc} */
+    @Override public void testReplicated() throws Exception {
+        super.testReplicated();
+
+        checkReadFromBackupUserTxs();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkReadFromBackupUserTxs() throws Exception {
+        checkReadFromBackup(OPTIMISTIC, READ_COMMITTED);
+        checkReadFromBackup(OPTIMISTIC, REPEATABLE_READ);
+        checkReadFromBackup(OPTIMISTIC, SERIALIZABLE);
+
+        checkReadFromBackup(PESSIMISTIC, READ_COMMITTED);
+        checkReadFromBackup(PESSIMISTIC, REPEATABLE_READ);
+        checkReadFromBackup(PESSIMISTIC, SERIALIZABLE);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void testPartitioned() throws Exception {
+        super.testPartitioned();
+
+        checkReadFromBackupUserTxs();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void testNearReplicated() throws Exception {
+        super.testNearReplicated();
+
+        checkReadFromBackupUserTxs();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void testNearPartitioned() throws Exception {
+        super.testNearPartitioned();
+
+        checkReadFromBackupUserTxs();
+    }
 }

From fe75a61368d914858216fb0e39d1af84ee93b3b2 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 30 Sep 2016 13:27:53 +0300
Subject: [PATCH 463/516] Fixed ByteBufferNioClientWorker/ssl write according
 to recent changes in nio session.

(cherry picked from commit a8ef28e)
---
 .../internal/util/nio/GridNioServer.java      | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index c3208f9d9ec0b..02118374ee4f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -985,12 +985,12 @@ else if (cnt == 0)
 
                     if (req == null) {
                         if (ses.procWrite.get()) {
-                            ses.procWrite.set(false);
+                            boolean set = ses.procWrite.compareAndSet(true, false);
 
-                            if (ses.writeQueue().isEmpty()) {
-                                if ((key.interestOps() & SelectionKey.OP_WRITE) != 0)
-                                    key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
-                            }
+                            assert set;
+
+                            if (ses.writeQueue().isEmpty())
+                                key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
                             else
                                 ses.procWrite.set(true);
                         }
@@ -1206,17 +1206,17 @@ private void processWriteSsl(SelectionKey key) throws IOException {
                         if (req == null) {
                             req = ses.pollFuture();
 
-                            if (req == null && buf.position() == 0) {
-                                if (ses.procWrite.get()) {
-                                    ses.procWrite.set(false);
+                        if (req == null && buf.position() == 0) {
+                            if (ses.procWrite.get()) {
+                                boolean set = ses.procWrite.compareAndSet(true, false);
 
-                                    if (ses.writeQueue().isEmpty()) {
-                                        if ((key.interestOps() & SelectionKey.OP_WRITE) != 0)
-                                            key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
-                                    }
-                                    else
-                                        ses.procWrite.set(true);
-                                }
+                                assert set;
+
+                                if (ses.writeQueue().isEmpty())
+                                    key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+                                else
+                                    ses.procWrite.set(true);
+                            }
 
                                 break;
                             }

From dcf473bab4466e294c2d788b710e53e375bc11b4 Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Tue, 26 Dec 2017 17:02:26 +0300
Subject: [PATCH 464/516] GG-13139: increased buffer sizes in
 IgniteCacheMessageWriteTimeoutTest to match values in apache master

---
 .../distributed/IgniteCacheMessageWriteTimeoutTest.java     | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index 46055251cc469..da23d1bae8d40 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -50,8 +50,8 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
         // Try provoke connection close on socket writeTimeout.
         commSpi.setSharedMemoryPort(-1);
         commSpi.setMessageQueueLimit(10);
-        commSpi.setSocketReceiveBuffer(40);
-        commSpi.setSocketSendBuffer(40);
+        commSpi.setSocketReceiveBuffer(64);
+        commSpi.setSocketSendBuffer(64);
         commSpi.setSocketWriteTimeout(100);
         commSpi.setUnacknowledgedMessagesBufferSize(1000);
         commSpi.setConnectTimeout(10_000);
@@ -75,8 +75,6 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testMessageQueueLimit() throws Exception {
-        fail("https://ggsystems.atlassian.net/browse/GG-13139");
-
         for (int i = 0; i < 15; i++) {
             log.info("Iteration: " + i);
 

From f3ec31a5c1995655315d6869466d0c89ac05d7c2 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dgovorukhin@gridgain.com>
Date: Mon, 20 Feb 2017 18:23:58 +0300
Subject: [PATCH 465/516] GG-13014: Fixed flaky
 GridCacheInterceptorAtomicRebalanceTest.testGetAndPut.

ignite-db-x more logging, rebalance interceptor test refactoring
(cherry picked from commit 5eeaa38)
---
 .../apache/ignite/internal/IgniteKernal.java  |  3 +++
 ...AbstractCacheInterceptorRebalanceTest.java | 27 ++++++++++++-------
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 81c9e548ced4b..17d696acce307 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -939,6 +939,9 @@ public void start(
                 fillNodeAttributes(clusterProc.updateNotifierEnabled());
             }
             catch (Throwable e) {
+                U.error(
+                    log, "Exception during start processors, node will be stopped and close connections", e);
+
                 // Stop discovery spi to close tcp socket.
                 ctx.discovery().stop(true);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
index 3a2bc81643840..391cd637d3169 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
@@ -79,6 +79,8 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
 
         final CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>(CACHE_NAME);
 
+        assertNotNull(interceptor);
+
         ccfg.setInterceptor(interceptor);
         ccfg.setAtomicityMode(atomicityMode());
         ccfg.setWriteSynchronizationMode(FULL_SYNC);
@@ -107,6 +109,8 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
         stopAllGrids();
 
         super.afterTest();
+
+        interceptor = null;
     }
 
     /**
@@ -198,8 +202,6 @@ public void testGetAndPut() throws Exception {
      * @throws Exception If fail.
      */
     private void testRebalance(final Operation operation) throws Exception {
-        interceptor = new RebalanceUpdateInterceptor();
-
         long stopTime = System.currentTimeMillis() + 2 * 60_000;
 
         for (int iter = 0; iter < TEST_ITERATIONS && System.currentTimeMillis() < stopTime; iter++) {
@@ -277,8 +279,10 @@ private interface Operation {
      */
     private static class UpdateEntryProcessor implements EntryProcessor<Integer, Integer, Integer> {
         /** {@inheritDoc} */
-        @Override public Integer process(final MutableEntry<Integer, Integer> entry,
-            final Object... arguments) throws EntryProcessorException {
+        @Override public Integer process(
+            final MutableEntry<Integer, Integer> entry,
+            final Object... arguments
+        ) throws EntryProcessorException {
             entry.setValue((Integer) arguments[0]);
 
             return null;
@@ -290,8 +294,10 @@ private static class UpdateEntryProcessor implements EntryProcessor<Integer, Int
      */
     private static class RemoveEntryProcessor implements EntryProcessor<Integer, Integer, Integer> {
         /** {@inheritDoc} */
-        @Override public Integer process(final MutableEntry<Integer, Integer> entry,
-            final Object... arguments) throws EntryProcessorException {
+        @Override public Integer process(
+            final MutableEntry<Integer, Integer> entry,
+            final Object... arguments
+        ) throws EntryProcessorException {
             entry.remove();
 
             return null;
@@ -306,7 +312,10 @@ private static class RebalanceUpdateInterceptor extends CacheInterceptorAdapter<
         private static final long serialVersionUID = 0L;
 
         /** {@inheritDoc} */
-        @Nullable @Override public Integer onBeforePut(final Cache.Entry entry, final Integer newVal) {
+        @Nullable @Override public Integer onBeforePut(
+            final Cache.Entry entry,
+            final Integer newVal
+        ) {
             try {
                 boolean first = entry.getKey().equals(newVal);
 
@@ -339,7 +348,8 @@ private static class RebalanceRemoveInterceptor extends CacheInterceptorAdapter<
 
         /** {@inheritDoc} */
         @Nullable @Override public IgniteBiTuple<Boolean, Integer> onBeforeRemove(
-            final Cache.Entry<Integer, Integer> entry) {
+            final Cache.Entry<Integer, Integer> entry
+        ) {
             try {
                 assertNotNull("Null old value: " + entry, entry.getValue());
                 assertEquals("Unexpected old value: " + entry, entry.getKey(), entry.getValue());
@@ -354,5 +364,4 @@ private static class RebalanceRemoveInterceptor extends CacheInterceptorAdapter<
             return new T2<>(true, null);
         }
     }
-
 }

From 5fdd14244d1b627592e1ef603087c305aac2c8d1 Mon Sep 17 00:00:00 2001
From: apopov <tank2.alex@gmail.com>
Date: Fri, 29 Dec 2017 17:01:31 +0300
Subject: [PATCH 466/516] GG-13272 Fix [GridClockSyncProcessor] Failed to send
 time request to remote node

---
 .../service/GridServiceProcessorAbstractSelfTest.java           | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
index debe4859d1c19..bfb5dd0e63915 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorAbstractSelfTest.java
@@ -85,6 +85,8 @@ public abstract class GridServiceProcessorAbstractSelfTest extends GridCommonAbs
 
         c.setCacheConfiguration(cc);
 
+        c.setLocalHost("127.0.0.1");
+
         return c;
     }
 

From 5b27a8e08a5d0cc4d5b3566646ac128cc1da4c90 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@gridgain.com>
Date: Fri, 7 Jul 2017 13:14:18 +0300
Subject: [PATCH 467/516] 2.1 More simple tx cancel on node stop

(cherry picked from commit ab52671)

Fixes GG-12940
---
 .../processors/cache/GridCacheProcessor.java  |  2 ++
 .../dht/GridDhtTxFinishFuture.java            |  6 +++-
 .../near/GridNearTxFinishFuture.java          | 12 +++++--
 .../cache/transactions/IgniteTxAdapter.java   | 34 +++++++++++--------
 .../transactions/IgniteTxLocalAdapter.java    | 15 ++++----
 5 files changed, 45 insertions(+), 24 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index e1a497ffd14a4..b3b97fcd1153e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -953,6 +953,8 @@ public void blockGateways() {
         // No new caches should be added after this point.
         exch.onKernalStop(cancel);
 
+        sharedCtx.mvcc().onStop();
+
         for (GridCacheAdapter<?, ?> cache : caches.values()) {
             GridCacheAffinityManager aff = cache.context().affinity();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index 60e07b37ba822..d0635e9ab23ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.cache.GridCacheFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
@@ -39,6 +40,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
@@ -217,7 +219,9 @@ public void onResult(UUID nodeId, GridDhtTxFinishResponse res) {
 
             if (this.tx.onePhaseCommit() && (this.tx.state() == COMMITTING)) {
                 try {
-                    this.tx.tmFinish(err == null);
+                    boolean nodeStop = err != null && X.hasCause(err, NodeStoppingException.class);
+
+                    this.tx.tmFinish(err == null, nodeStop);
                 }
                 catch (IgniteCheckedException finishErr) {
                     U.error(log, "Failed to finish tx: " + tx, e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 903bb8f1e9368..6d193637fbb4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -298,9 +299,14 @@ else if (fut.getClass() == CheckRemoteTxMiniFuture.class) {
             if (isDone())
                 return false;
 
-            if (err != null)
+            boolean nodeStop = false;
+
+            if (err != null) {
                 tx.setRollbackOnly();
 
+                nodeStop = err instanceof NodeStoppingException;
+            }
+
             if (commit) {
                 if (tx.commitError() != null)
                     err = tx.commitError();
@@ -332,7 +338,7 @@ else if (err != null)
                     finishOnePhase(commit);
 
                     try {
-                        tx.tmFinish(commit);
+                        tx.tmFinish(commit, nodeStop);
                     }
                     catch (IgniteCheckedException e) {
                         U.error(log, "Failed to finish tx: " + tx, e);
@@ -343,7 +349,7 @@ else if (err != null)
                 }
 
                 if (super.onDone(tx0, err)) {
-                    if (error() instanceof IgniteTxHeuristicCheckedException) {
+                    if (error() instanceof IgniteTxHeuristicCheckedException && !nodeStop) {
                         AffinityTopologyVersion topVer = tx.topologyVersion();
 
                         for (IgniteTxEntry e : tx.writeMap().values()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 1a84a78ab6371..88ea5461ba593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -434,26 +434,32 @@ public void storeEnabled(boolean storeEnabled) {
      */
     @SuppressWarnings({"CatchGenericClass"})
     protected void uncommit(boolean nodeStopping) {
-        if (!nodeStopping) {
-            for (IgniteTxEntry e : writeMap().values()) {
-                try {
-                    GridCacheEntryEx entry = e.cached();
+        try {
+            if (!nodeStopping) {
+                for (IgniteTxEntry e : writeMap().values()) {
+                    try {
+                        GridCacheEntryEx entry = e.cached();
+
+                        if (e.op() != NOOP)
+                            entry.invalidate(xidVer);
+                    }
+                    catch (Throwable t) {
+                        U.error(log, "Failed to invalidate transaction entries while reverting a commit.", t);
 
-                    if (e.op() != NOOP)
-                        entry.invalidate(xidVer);
-                }
-                catch (Throwable t) {
-                    U.error(log, "Failed to invalidate transaction entries while reverting a commit.", t);
+                        if (t instanceof Error)
+                            throw (Error)t;
+
+                            break;
+                        }
+                    }
 
-                    if (t instanceof Error)
-                        throw (Error)t;
 
-                    break;
+                    cctx.tm().uncommitTx(this);
                 }
             }
+        catch (Exception ex) {
+            U.error(log, "Failed to do uncommit.", ex);
         }
-
-        cctx.tm().uncommitTx(this);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 92893c15464c1..55c30459844a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1017,17 +1017,20 @@ assert ownsLock(txEntry.cached()):
      * Commits transaction to transaction manager. Used for one-phase commit transactions only.
      *
      * @param commit If {@code true} commits transaction, otherwise rollbacks.
+     * @param nodeStop If {@code true} tx is cancelled on node stop.
      * @throws IgniteCheckedException If failed.
      */
-    public void tmFinish(boolean commit) throws IgniteCheckedException {
+    public void tmFinish(boolean commit, boolean nodeStop) throws IgniteCheckedException {
         assert onePhaseCommit();
 
         if (DONE_FLAG_UPD.compareAndSet(this, 0, 1)) {
-            // Unlock all locks.
-            if (commit)
-                cctx.tm().commitTx(this);
-            else
-                cctx.tm().rollbackTx(this);
+            if (!nodeStop) {
+                // Unlock all locks.
+                if (commit)
+                    cctx.tm().commitTx(this);
+                else
+                    cctx.tm().rollbackTx(this);
+            }
 
             state(commit ? COMMITTED : ROLLED_BACK);
 

From 1c37222bb02f1a98600349f15dbdd653cdd630a5 Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Wed, 10 Jan 2018 11:53:47 +0300
Subject: [PATCH 468/516] Mute
 IgniteCacheMessageWriteTimeoutTest.testMessageQueueLimit

---
 .../cache/distributed/IgniteCacheMessageWriteTimeoutTest.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index da23d1bae8d40..06585100ae005 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -75,6 +75,8 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testMessageQueueLimit() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-13139");
+
         for (int i = 0; i < 15; i++) {
             log.info("Iteration: " + i);
 

From f4caf3058045917069f2e99152804f117cdd7231 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 11 Jan 2018 16:35:58 +0300
Subject: [PATCH 469/516] GG-13140 - Fix
 GridCacheDhtEvictionSelfTest.testMultipleKeys

---
 .../cache/distributed/dht/GridCacheDhtEvictionSelfTest.java  | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEvictionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEvictionSelfTest.java
index c55f2985d75b7..765d178f99046 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEvictionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEvictionSelfTest.java
@@ -321,6 +321,11 @@ public void testMultipleKeys() throws Exception {
             }
         }, EVT_CACHE_ENTRY_EVICTED);
 
+        // Need to evict on stable topology, otherwise primary eviction
+        // may be ignored (GridCacheEvictionManager.onFutureCompleted()).
+        // Current eviction implementation doesn't have guarantees on successful eviction.
+        awaitPartitionMapExchange();
+
         // Evict on primary node.
         // Eviction of the last key should trigger queue processing.
         for (Integer key : keys)

From 734fd9e81300c15ce4222d65ef6d39f5aa264039 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 11 Jan 2018 16:36:39 +0300
Subject: [PATCH 470/516] ignite-7239 In case of not serializable cache update
 response, future on node requester will never complete

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit 65e1c57)
---
 .../cache/CacheInvokeDirectResult.java        |  16 +-
 .../IgniteCacheFailedUpdateResponseTest.java  | 310 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   4 +-
 3 files changed, 327 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
index cc453f919a403..f708f82cd4f97 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
@@ -105,8 +105,20 @@ public CacheObject result() {
     public void prepareMarshal(GridCacheContext ctx) throws IgniteCheckedException {
         key.prepareMarshal(ctx.cacheObjectContext());
 
-        if (err != null && errBytes == null)
-            errBytes = U.marshal(ctx.marshaller(), err);
+        if (err != null && errBytes == null) {
+            try {
+                errBytes = U.marshal(ctx.marshaller(), err);
+            }
+            catch (IgniteCheckedException e) {
+                // Try send exception even if it's unable to marshal.
+                IgniteCheckedException exc = new IgniteCheckedException(err.getMessage());
+
+                exc.setStackTrace(err.getStackTrace());
+                exc.addSuppressed(e);
+
+                errBytes = U.marshal(ctx.marshaller(), exc);
+            }
+        }
 
         if (res != null)
             res.prepareMarshal(ctx.cacheObjectContext());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java
new file mode 100644
index 0000000000000..ebcff7c43898e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.NotSerializableException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.cache.CachePartialUpdateException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
+
+/**
+ * Checks that no future hangs on non-srializable exceptions and values.
+ */
+public class IgniteCacheFailedUpdateResponseTest extends GridCommonAbstractTest {
+    /** Atomic cache. */
+    private static final String ATOMIC_CACHE = "atomic";
+
+    /** Tx cache. */
+    private static final String TX_CACHE = "tx";
+
+    /** Atomic cache. */
+    private IgniteCache<Object, Object> atomicCache;
+
+    /** Tx cache. */
+    private IgniteCache<Object, Object> txCache;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        CacheConfiguration atomicCfg = new CacheConfiguration(ATOMIC_CACHE);
+        CacheConfiguration txCfg = new CacheConfiguration(TX_CACHE);
+
+        atomicCfg.setBackups(1);
+        txCfg.setBackups(1);
+
+        txCfg.setAtomicityMode(TRANSACTIONAL);
+
+        cfg.setCacheConfiguration(atomicCfg, txCfg);
+
+        cfg.setClientMode(igniteInstanceName.contains("client"));
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(LOCAL_IP_FINDER);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrid("server-1");
+        startGrid("server-2");
+        startGrid("client");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        atomicCache = grid("client").cache(ATOMIC_CACHE);
+        txCache = grid("client").cache(TX_CACHE);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testInvokeAtomic() throws Exception {
+        testInvoke(atomicCache);
+        testInvokeAll(atomicCache);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testInvokeTx() throws Exception {
+        testInvoke(txCache);
+        testInvokeAll(txCache);
+
+        IgniteEx client = grid("client");
+
+        Callable<Object> clos = new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                testInvoke(txCache);
+                testInvokeAll(txCache);
+
+                return null;
+            }
+        };
+
+        doInTransaction(client, PESSIMISTIC, READ_COMMITTED, clos);
+        doInTransaction(client, PESSIMISTIC, REPEATABLE_READ, clos);
+        doInTransaction(client, PESSIMISTIC, SERIALIZABLE, clos);
+        doInTransaction(client, OPTIMISTIC, READ_COMMITTED, clos);
+        doInTransaction(client, OPTIMISTIC, REPEATABLE_READ, clos);
+        doInTransaction(client, OPTIMISTIC, SERIALIZABLE, clos);
+    }
+
+    /**
+     * @param cache Cache.
+     */
+    private void testInvoke(final IgniteCache<Object, Object> cache) throws Exception {
+        Class<? extends Exception> exp = grid("client").transactions().tx() == null
+            ? EntryProcessorException.class
+            : NonSerializableException.class;
+
+        //noinspection ThrowableNotThrown
+        assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                cache.invoke("1", new UpdateProcessor());
+
+                return null;
+            }
+        }, exp, null);
+
+        if (ATOMIC_CACHE.equals(cache.getName())) {
+            //noinspection ThrowableNotThrown
+            assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    cache.invoke("1", new UpdateValueProcessor());
+
+                    return null;
+                }
+            }, CachePartialUpdateException.class, null);
+        }
+    }
+
+    /**
+     * @param cache Cache.
+     */
+    private void testInvokeAll(final IgniteCache<Object, Object> cache) throws Exception {
+        Map<Object, EntryProcessorResult<Object>> results = cache.invokeAll(Collections.singleton("1"), new UpdateProcessor());
+
+        final EntryProcessorResult<Object> epRes = F.first(results.values());
+
+        assertNotNull(epRes);
+
+        // In transactions EP will be invoked locally.
+        Class<? extends Exception> exp = grid("client").transactions().tx() == null
+            ? EntryProcessorException.class
+            : NonSerializableException.class;
+
+        //noinspection ThrowableNotThrown
+        assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                epRes.get();
+
+                return null;
+            }
+        }, exp, null);
+
+        if (ATOMIC_CACHE.equals(cache.getName())) {
+            //noinspection ThrowableNotThrown
+            assertThrows(log, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    cache.invokeAll(Collections.singleton("1"), new UpdateValueProcessor());
+
+                    return null;
+                }
+            }, CachePartialUpdateException.class, null);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Value implements Externalizable, Binarylizable {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         *
+         */
+        public Value() {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(ObjectOutput out) throws IOException {
+            throw new NotSerializableException("Test marshalling exception");
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+            throw new BinaryObjectException("Test marshalling exception");
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+            // No-op.
+        }
+    }
+
+    /**
+     *
+     */
+    private static class NonSerializableException extends EntryProcessorException implements Externalizable, Binarylizable {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         *
+         */
+        public NonSerializableException() {
+            super();
+        }
+
+        /**
+         * @param msg Message.
+         */
+        NonSerializableException(String msg) {
+            super(msg);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(ObjectOutput out) throws IOException {
+            throw new NotSerializableException("Test marshalling exception");
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+            throw new BinaryObjectException("Test marshalling exception");
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+            // No-op.
+        }
+    }
+
+    /**
+     *
+     */
+    private static class UpdateProcessor implements CacheEntryProcessor<Object, Object, Object> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public Object process(MutableEntry<Object, Object> entry,
+            Object... arguments) throws EntryProcessorException {
+            throw new NonSerializableException("Test exception");
+        }
+    }
+
+    /**
+     *
+     */
+    private static class UpdateValueProcessor implements CacheEntryProcessor<Object, Object, Object> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public Object process(MutableEntry<Object, Object> entry,
+            Object... arguments) throws EntryProcessorException {
+            return new Value();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 20_000;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 94492fa9e0b2c..0fb44bd6297ac 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -106,11 +106,12 @@
 import org.apache.ignite.internal.processors.cache.distributed.CacheNoValueClassOnServerNodeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheCreatePutMultiNodeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheCreatePutTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheFailedUpdateResponseTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePrimarySyncTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheReadFromBackupTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheSingleGetMessageTest;
-import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCacheWriteSynchronizationModesMultithreadedTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCachePrimarySyncTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCacheWriteSynchronizationModesMultithreadedTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtTxPreloadSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheLockFailoverSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheMultiTxLockSelfTest;
@@ -246,6 +247,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheCreatePutMultiNodeSelfTest.class);
         suite.addTestSuite(IgniteCacheCreatePutTest.class);
         suite.addTestSuite(IgniteClientCacheInitializationFailTest.class);
+        suite.addTestSuite(IgniteCacheFailedUpdateResponseTest.class);
 
         suite.addTestSuite(GridCacheTxLoadFromStoreOnLockSelfTest.class);
 

From c4d3b4709c50547d1e8bb7ee52034113874bda04 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Thu, 11 Jan 2018 16:48:43 +0300
Subject: [PATCH 471/516] ignite-7340 Fix flaky
 GridServiceProcessorMultiNodeConfigSelfTest#checkDeployOnEachNodeUpdateTopology

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit 6136ecd)
---
 .../internal/processors/service/GridServiceProcessor.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 8914094be3d54..73c8ac39463b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1352,6 +1352,14 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
      * @param assigns Assignments.
      */
     private void redeploy(GridServiceAssignments assigns) {
+        if (assigns.topologyVersion() < ctx.discovery().topologyVersion()) {
+            if (log.isDebugEnabled())
+                log.debug("Skip outdated assignment [assigns=" + assigns +
+                    ", topVer=" + ctx.discovery().topologyVersion() + ']');
+
+            return;
+        }
+
         String svcName = assigns.name();
 
         Integer assignCnt = assigns.assigns().get(ctx.localNodeId());

From ef39ae5881363fc9ed29394ba21bae628d0c8239 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 15 Jan 2018 09:16:50 +0300
Subject: [PATCH 472/516] ignite-7239 - Mods for 1.x

---
 .../cache/CacheInvokeDirectResult.java        | 28 +++++++++++--
 .../IgniteCacheFailedUpdateResponseTest.java  | 40 +++++++++++--------
 2 files changed, 49 insertions(+), 19 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
index f708f82cd4f97..bc555a1470594 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
@@ -105,6 +105,31 @@ public CacheObject result() {
     public void prepareMarshal(GridCacheContext ctx) throws IgniteCheckedException {
         key.prepareMarshal(ctx.cacheObjectContext());
 
+        marshalError(ctx);
+
+        if (res != null) {
+            try {
+                res.prepareMarshal(ctx.cacheObjectContext());
+            }
+            catch (IgniteCheckedException e) {
+                if (err != null)
+                    err.addSuppressed(e);
+                else
+                    err = e;
+
+                errBytes = null;
+                res = null;
+
+                marshalError(ctx);
+            }
+        }
+    }
+
+    /**
+     * @param ctx Context.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void marshalError(GridCacheContext ctx) throws IgniteCheckedException {
         if (err != null && errBytes == null) {
             try {
                 errBytes = U.marshal(ctx.marshaller(), err);
@@ -119,9 +144,6 @@ public void prepareMarshal(GridCacheContext ctx) throws IgniteCheckedException {
                 errBytes = U.marshal(ctx.marshaller(), exc);
             }
         }
-
-        if (res != null)
-            res.prepareMarshal(ctx.cacheObjectContext());
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java
index ebcff7c43898e..e6c70a747e906 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java
@@ -84,6 +84,8 @@ public class IgniteCacheFailedUpdateResponseTest extends GridCommonAbstractTest
 
         ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(LOCAL_IP_FINDER);
 
+        cfg.setMarshaller(null);
+
         return cfg;
     }
 
@@ -152,14 +154,14 @@ private void testInvoke(final IgniteCache<Object, Object> cache) throws Exceptio
         }, exp, null);
 
         if (ATOMIC_CACHE.equals(cache.getName())) {
-            //noinspection ThrowableNotThrown
-            assertThrows(log, new Callable<Object>() {
-                @Override public Object call() throws Exception {
-                    cache.invoke("1", new UpdateValueProcessor());
-
-                    return null;
-                }
-            }, CachePartialUpdateException.class, null);
+            try {
+                cache.invoke("1", new UpdateValueProcessor());
+
+                fail("Exception must be thrown");
+            }
+            catch (CachePartialUpdateException | EntryProcessorException e) {
+                e.printStackTrace(System.out);
+            }
         }
     }
 
@@ -188,14 +190,20 @@ private void testInvokeAll(final IgniteCache<Object, Object> cache) throws Excep
         }, exp, null);
 
         if (ATOMIC_CACHE.equals(cache.getName())) {
-            //noinspection ThrowableNotThrown
-            assertThrows(log, new Callable<Object>() {
-                @Override public Object call() throws Exception {
-                    cache.invokeAll(Collections.singleton("1"), new UpdateValueProcessor());
-
-                    return null;
-                }
-            }, CachePartialUpdateException.class, null);
+            try {
+                Map<Object, EntryProcessorResult<Object>> resMap = cache.invokeAll(Collections.singleton("1"), new UpdateValueProcessor());
+
+                EntryProcessorResult<Object> res = F.first(resMap.values());
+
+                assertNotNull(res);
+
+                res.get();
+
+                fail("Exception must be thrown");
+            }
+            catch (CachePartialUpdateException | EntryProcessorException e) {
+                e.printStackTrace(System.out);
+            }
         }
     }
 

From 2a4a2679853dc16288608dd7e5fdced96be1ef4a Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Mon, 15 Jan 2018 12:01:19 +0300
Subject: [PATCH 473/516] GG-12702 - Mute tests. GG-11241 must be fixed first.

(cherry picked from commit 44badf3)
---
 .../ttl/CacheTtlOnheapAtomicPartitionedSelfTest.java | 12 +++++++++++-
 ...cheTtlOnheapTransactionalPartitionedSelfTest.java | 10 ++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlOnheapAtomicPartitionedSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlOnheapAtomicPartitionedSelfTest.java
index dc2dc950281b5..a1b2d53896146 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlOnheapAtomicPartitionedSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlOnheapAtomicPartitionedSelfTest.java
@@ -16,7 +16,7 @@
  */
 package org.apache.ignite.internal.processors.cache.ttl;
 
-import org.apache.ignite.cache.*;
+import org.apache.ignite.cache.CacheMode;
 
 /**
  * TTL test with offheap.
@@ -31,4 +31,14 @@ public class CacheTtlOnheapAtomicPartitionedSelfTest extends CacheTtlOnheapAtomi
     @Override protected int gridCount() {
         return 2;
     }
+
+    /** {@inheritDoc} */
+    @Override public void testDefaultTimeToLiveLoadAll() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-11241");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void testDefaultTimeToLiveStreamerAdd() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-11241");
+    }
 }
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlOnheapTransactionalPartitionedSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlOnheapTransactionalPartitionedSelfTest.java
index 70fbb727d5a3f..2d9beb72f58e2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlOnheapTransactionalPartitionedSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ttl/CacheTtlOnheapTransactionalPartitionedSelfTest.java
@@ -31,4 +31,14 @@ public class CacheTtlOnheapTransactionalPartitionedSelfTest extends CacheTtlOnhe
     @Override protected int gridCount() {
         return 2;
     }
+
+    /** {@inheritDoc} */
+    @Override public void testDefaultTimeToLiveLoadAll() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-11241");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void testDefaultTimeToLiveStreamerAdd() throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-11241");
+    }
 }
\ No newline at end of file

From 13b994538986d5670d1cc97bf6981013b7f1fada Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Tue, 16 Jan 2018 17:47:58 +0300
Subject: [PATCH 474/516] ignite-7334 Added escaping of special symbols during
 MBean initialization

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit 536009b)
---
 .../processors/cache/GridCacheProcessor.java  |  4 +-
 .../ignite/internal/util/IgniteUtils.java     | 91 ++++---------------
 .../testsuites/IgniteUtilSelfTestSuite.java   |  4 +
 .../util/mbeans/GridMBeanDisableSelfTest.java | 69 +++++++-------
 .../mbeans/GridMBeanExoticNamesSelfTest.java  | 70 ++++++++++++++
 5 files changed, 125 insertions(+), 113 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanExoticNamesSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index b3b97fcd1153e..06f3ccfbb1fb3 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3635,7 +3635,7 @@ private void registerMbean(Object o, @Nullable String cacheName, boolean near)
         for (Class<?> itf : o.getClass().getInterfaces()) {
             if (itf.getName().endsWith("MBean") || itf.getName().endsWith("MXBean")) {
                 try {
-                    U.registerCacheMBean(srvr, ctx.gridName(), cacheName, o.getClass().getName(), o,
+                    U.registerMBean(srvr, ctx.gridName(), cacheName, o.getClass().getName(), o,
                         (Class<Object>)itf);
                 }
                 catch (Throwable e) {
@@ -3671,7 +3671,7 @@ private void unregisterMbean(Object o, @Nullable String cacheName, boolean near)
         for (Class<?> itf : o.getClass().getInterfaces()) {
             if (itf.getName().endsWith("MBean") || itf.getName().endsWith("MXBean")) {
                 try {
-                    srvr.unregisterMBean(U.makeCacheMBeanName(ctx.gridName(), cacheName, o.getClass().getName()));
+                    srvr.unregisterMBean(U.makeMBeanName(ctx.gridName(), cacheName, o.getClass().getName()));
                 }
                 catch (Throwable e) {
                     U.error(log, "Failed to unregister MBean for component: " + o, e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 4d57212f6af39..79f91663e6334 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -4385,9 +4385,9 @@ public static ObjectName makeMBeanName(@Nullable String gridName, @Nullable Stri
             sb.a("grid=").a(gridName).a(',');
 
         if (grp != null)
-            sb.a("group=").a(grp).a(',');
+            sb.a("group=").a(escapeObjectNameValue(grp)).a(',');
 
-        sb.a("name=").a(name);
+        sb.a("name=").a(escapeObjectNameValue(name));
 
         return new ObjectName(sb.toString());
     }
@@ -4425,36 +4425,16 @@ public static String maskName(@Nullable String name) {
     }
 
     /**
-     * Constructs JMX object name with given properties.
-     * Map with ordered {@code groups} used for proper object name construction.
+     * Escapes the given string to be used as a value in the ObjectName syntax.
      *
-     * @param gridName Grid name.
-     * @param cacheName Name of the cache.
-     * @param name Name of mbean.
-     * @return JMX object name.
-     * @throws MalformedObjectNameException Thrown in case of any errors.
+     * @param s A string to be escape.
+     * @return An escaped string.
      */
-    public static ObjectName makeCacheMBeanName(@Nullable String gridName, @Nullable String cacheName, String name)
-        throws MalformedObjectNameException {
-        SB sb = new SB(JMX_DOMAIN + ':');
-
-        appendClassLoaderHash(sb);
-
-        appendJvmId(sb);
-
-        if (gridName != null && !gridName.isEmpty())
-            sb.a("grid=").a(gridName).a(',');
-
-        cacheName = maskName(cacheName);
-
-        if (!MBEAN_CACHE_NAME_PATTERN.matcher(cacheName).matches())
-            sb.a("group=").a('\"').a(cacheName).a('\"').a(',');
-        else
-            sb.a("group=").a(cacheName).a(',');
+    private static String escapeObjectNameValue(String s) {
+        if (MBEAN_CACHE_NAME_PATTERN.matcher(s).matches())
+            return s;
 
-        sb.a("name=").a(name);
-
-        return new ObjectName(sb.toString());
+        return '\"' + s.replaceAll("[\\\\\"?*]", "\\\\$0") + '\"';
     }
 
     /**
@@ -4462,7 +4442,7 @@ public static ObjectName makeCacheMBeanName(@Nullable String gridName, @Nullable
      *
      * @param <T> Type of mbean.
      * @param mbeanSrv MBean server.
-     * @param gridName Grid name.
+     * @param igniteInstanceName Grid name.
      * @param grp Name of the group.
      * @param name Name of mbean.
      * @param impl MBean implementation.
@@ -4471,19 +4451,14 @@ public static ObjectName makeCacheMBeanName(@Nullable String gridName, @Nullable
      * @throws MBeanRegistrationException if MBeans are disabled.
      * @throws JMException If MBean creation failed.
      */
-    public static <T> ObjectName registerMBean(MBeanServer mbeanSrv, @Nullable String gridName, @Nullable String grp,
-        String name, T impl, @Nullable Class<T> itf) throws JMException {
-        if(IGNITE_MBEANS_DISABLED)
-            throw new MBeanRegistrationException(new IgniteIllegalStateException("No MBeans are allowed."));
-        assert mbeanSrv != null;
-        assert name != null;
-        assert itf != null;
-
-        DynamicMBean mbean = new IgniteStandardMXBean(impl, itf);
-
-        mbean.getMBeanInfo();
-
-        return mbeanSrv.registerMBean(mbean, makeMBeanName(gridName, grp, name)).getObjectName();
+    public static <T> ObjectName registerMBean(
+        MBeanServer mbeanSrv,
+        @Nullable String igniteInstanceName,
+        @Nullable String grp,
+        String name, T impl,
+        @Nullable Class<T> itf
+    ) throws JMException {
+        return registerMBean(mbeanSrv, makeMBeanName(igniteInstanceName, grp, name), impl, itf);
     }
 
     /**
@@ -4514,36 +4489,6 @@ public static <T> ObjectName registerMBean(MBeanServer mbeanSrv, ObjectName name
         return mbeanSrv.registerMBean(mbean, name).getObjectName();
     }
 
-    /**
-     * Registers MBean with the server.
-     *
-     * @param <T> Type of mbean.
-     * @param mbeanSrv MBean server.
-     * @param gridName Grid name.
-     * @param cacheName Name of the cache.
-     * @param name Name of mbean.
-     * @param impl MBean implementation.
-     * @param itf MBean interface.
-     * @return JMX object name.
-     * @throws MBeanRegistrationException if MBeans are disabled.
-     * @throws JMException If MBean creation failed.
-     */
-    public static <T> ObjectName registerCacheMBean(MBeanServer mbeanSrv, @Nullable String gridName,
-        @Nullable String cacheName, String name, T impl, Class<T> itf) throws JMException {
-        if(IGNITE_MBEANS_DISABLED)
-            throw new MBeanRegistrationException(new IgniteIllegalStateException("MBeans are disabled."));
-
-        assert mbeanSrv != null;
-        assert name != null;
-        assert itf != null;
-
-        DynamicMBean mbean = new IgniteStandardMXBean(impl, itf);
-
-        mbean.getMBeanInfo();
-
-        return mbeanSrv.registerMBean(mbean, makeCacheMBeanName(gridName, cacheName, name)).getObjectName();
-    }
-
     /**
      * Convenience method that interrupts a given thread if it's not {@code null}.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index 078b865a7325c..67f567da0d229 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -42,6 +42,8 @@
 import org.apache.ignite.util.GridQueueSelfTest;
 import org.apache.ignite.util.GridSpinReadWriteLockSelfTest;
 import org.apache.ignite.util.GridStringBuilderFactorySelfTest;
+import org.apache.ignite.util.mbeans.GridMBeanDisableSelfTest;
+import org.apache.ignite.util.mbeans.GridMBeanExoticNamesSelfTest;
 import org.apache.ignite.util.mbeans.GridMBeanSelfTest;
 
 /**
@@ -73,6 +75,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridToStringBuilderSelfTest.class);
         suite.addTestSuite(GridByteArrayListSelfTest.class);
         suite.addTestSuite(GridMBeanSelfTest.class);
+        suite.addTestSuite(GridMBeanDisableSelfTest.class);
+        suite.addTestSuite(GridMBeanExoticNamesSelfTest.class);
         suite.addTestSuite(GridLongListSelfTest.class);
         suite.addTestSuite(GridCacheUtilsSelfTest.class);
         suite.addTestSuite(IgniteExceptionRegistrySelfTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java
index f08f58bd9258a..9a238325151db 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java
@@ -23,7 +23,6 @@
 import javax.management.ObjectName;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -55,56 +54,50 @@ public class GridMBeanDisableSelfTest extends GridCommonAbstractTest {
     public void testCorrectMBeanInfo() throws Exception {
         // Node should start and stopped with no errors.
         try (final Ignite ignite = startGrid(0)) {
+            final MBeanServer server = ignite.configuration().getMBeanServer();
 
-            // Cache should be created and closed with no errors.
-            try (IgniteCache cache = ignite.getOrCreateCache(new CacheConfiguration("MyCache"))) {
+            GridTestUtils.assertThrowsWithCause(
+                new Callable<Void>() {
+                    @Override public Void call() throws Exception {
+                        U.registerMBean(server, ignite.name(), "dummy", "DummyMbean1", new DummyMBeanImpl(), DummyMBean.class);
 
-                final MBeanServer server = ignite.configuration().getMBeanServer();
+                        return null;
+                    }
+                }, MBeanRegistrationException.class);
 
-                GridTestUtils.assertThrowsWithCause(
-                    new Callable<Void>() {
-                        @Override public Void call() throws Exception {
-                            U.registerMBean(server, ignite.name(), "dummy", "DummyMbean1", new DummyMBeanImpl(), DummyMBean.class);
+            GridTestUtils.assertThrowsWithCause(
+                new Callable<Void>() {
+                    @Override public Void call() throws Exception {
+                        ObjectName objName = U.makeMBeanName(
+                            ignite.name(),
+                            "dummy",
+                            "DummyMbean2"
+                        );
 
-                            return null;
+                        U.registerMBean(server, objName, new DummyMBeanImpl(), DummyMBean.class);
 
-                        }
-                    }, MBeanRegistrationException.class);
+                        return null;
 
-                GridTestUtils.assertThrowsWithCause(
-                    new Callable<Void>() {
-                        @Override public Void call() throws Exception {
-                            ObjectName objName = U.makeMBeanName(
-                                ignite.name(),
-                                "dummy",
-                                "DummyMbean2"
-                            );
-
-                            U.registerMBean(server, objName, new DummyMBeanImpl(), DummyMBean.class);
-
-                            return null;
-
-                        }
-                    }, MBeanRegistrationException.class);
-
-                GridTestUtils.assertThrowsWithCause(
-                    new Callable<Void>() {
-                        @Override public Void call() throws Exception {
-                            U.registerCacheMBean(server, ignite.name(), "MyCache", "DummyMbean3",
-                                new DummyMBeanImpl(), DummyMBean.class);
-
-                            return null;
+                    }
+                }, MBeanRegistrationException.class);
+        }
+    }
 
-                        }
-                    }, MBeanRegistrationException.class);
-            }
+    /** Check that a cache can be started when MBeans are disabled. */
+    public void testCacheStart() throws Exception {
+        try (
+            Ignite ignite = startGrid(0);
+            IgniteCache<String, String> cache = ignite.getOrCreateCache("MyCache")
+        ) {
+            cache.put("foo", "bar");
+            assertEquals("bar", cache.get("foo"));
         }
     }
 
     /**
      * MBean dummy interface.
      */
-    interface DummyMBean {
+    public interface DummyMBean {
         /** */
         void noop();
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanExoticNamesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanExoticNamesSelfTest.java
new file mode 100644
index 0000000000000..3b79f24b1a2b0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanExoticNamesSelfTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util.mbeans;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Testing registration of MBeans with special characters in group name or bean name.
+ */
+public class GridMBeanExoticNamesSelfTest extends GridCommonAbstractTest {
+    /** Test registration of a bean with special characters in group name. */
+    public void testGroupWithSpecialSymbols() throws Exception {
+        checkMBeanRegistration("dummy!@#$^&*()?\\grp", "dummy");
+    }
+
+    /** Test registration of a bean with special characters in name. */
+    public void testNameWithSpecialSymbols() throws Exception {
+        checkMBeanRegistration("dummygrp", "dum!@#$^&*()?\\my");
+    }
+
+    /** Test MBean registration. */
+    private void checkMBeanRegistration(String grp, String name) throws Exception {
+        // Node should start and stop with no errors.
+        try (Ignite ignite = startGrid(0)) {
+            MBeanServer srv = ignite.configuration().getMBeanServer();
+
+            U.registerMBean(srv, ignite.name(), grp, name, new DummyMBeanImpl(), DummyMBean.class);
+
+            ObjectName objName = U.makeMBeanName(ignite.name(), grp, name + '2');
+            U.registerMBean(srv, objName, new DummyMBeanImpl(), DummyMBean.class);
+        }
+    }
+
+    /**
+     * MBean dummy interface.
+     */
+    public interface DummyMBean {
+        /** */
+        void noop();
+    }
+
+    /**
+     * MBean stub.
+     */
+    private static class DummyMBeanImpl implements DummyMBean {
+        /** {@inheritDoc} */
+        @Override public void noop() {
+            // No op.
+        }
+    }
+}
\ No newline at end of file

From 04fef454ef9b602990fd6b321da8671aaeab2cdf Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Sun, 14 Jan 2018 19:06:36 +0300
Subject: [PATCH 475/516] GG-13239 FairAffinityFunction does not distribute
 partitions properly with backup filter

---
 .../affinity/fair/FairAffinityFunction.java   |  71 ++++++-
 ...inityFunctionRechecksBackupFilterTest.java | 180 ++++++++++++++++++
 ...finityFunctionBackupFilterRecheckTest.java |  35 ++++
 .../testsuites/IgniteCacheTestSuite.java      |   2 +
 4 files changed, 285 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionRechecksBackupFilterTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunctionBackupFilterRecheckTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
index fe146ea7ad191..651e5f18a0f89 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
@@ -330,6 +330,9 @@ public void setCeilIdealPartitionsCount(boolean ceilIdealPartCnt) {
         // Per tier pending partitions.
         Map<Integer, Queue<Integer>> pendingParts = new HashMap<>();
 
+        for (int i = 0; i < assignment.size(); i++)
+            assignment.set(i, applyAffinityBackupFilterNodes(assignment.get(i)));
+
         FullAssignmentMap fullMap = new FullAssignmentMap(tiers, assignment, topSnapshot, neighborhoodMap);
 
         for (int tier = 0; tier < tiers; tier++) {
@@ -369,6 +372,22 @@ public void setCeilIdealPartitionsCount(boolean ceilIdealPartCnt) {
         return fullMap.assignments;
     }
 
+    private List<ClusterNode> applyAffinityBackupFilterNodes(List<ClusterNode> nodes) {
+        if(affinityBackupFilter == null || nodes.size() == 1)
+            return nodes;
+
+        ArrayList<ClusterNode> newList = new ArrayList<>(nodes.size());
+
+        newList.add(nodes.get(0));
+
+        for (int i = 1; i < nodes.size(); i++) {
+            if(affinityBackupFilter.apply(nodes.get(i), newList))
+                newList.add(nodes.get(i));
+        }
+
+        return newList;
+    }
+
     /** {@inheritDoc} */
     @Override public void reset() {
         // No-op.
@@ -537,8 +556,28 @@ private void assignPendingToNodes(int tier, Map<Integer, Queue<Integer>> pending
                 } while (i != idx);
             }
 
+            if (!assigned && (!exclNeighbors || exclNeighbors && allowNeighbors)) {
+                log.warning("Failed to find assignable node for partition. Partition is assigned ignoring affinity backup filter.");
+
+                do {
+                    ClusterNode node = topSnapshot.get(i);
+
+                    if (fullMap.assign(part, tier, node, pendingMap, false, allowNeighbors, true)) {
+                        it.remove();
+
+                        assigned = true;
+                    }
+
+                    i = (i + 1) % topSnapshot.size();
+
+                    if (assigned)
+                        idx = i;
+                } while (i != idx);
+            }
+
             if (!assigned && (!exclNeighbors || exclNeighbors && allowNeighbors))
                 throw new IllegalStateException("Failed to find assignable node for partition.");
+
         }
     }
 
@@ -845,10 +884,34 @@ boolean assign(int part,
             ClusterNode node,
             Map<Integer, Queue<Integer>> pendingParts, boolean force,
             boolean allowNeighbors)
+        {
+            return assign(part, tier, node, pendingParts,force, allowNeighbors, false);
+        }
+
+        /**
+         * Tries to assign partition to given node on specified tier. If force is false, assignment will succeed
+         * only if this partition is not already assigned to a node. If force is true, then assignment will succeed
+         * only if partition is not assigned to a tier with number less than passed in. Assigned partition from
+         * greater tier will be moved to pending queue.
+         *
+         * @param part Partition to assign.
+         * @param tier Tier number to assign.
+         * @param node Node to move partition to.
+         * @param pendingParts per tier pending partitions map.
+         * @param force Force flag.
+         * @param allowNeighbors Allow neighbors nodes for partition.
+         * @param ignoreFilter assign partition ignoring backup affinity filter.
+         * @return {@code True} if assignment succeeded.
+         */
+        boolean assign(int part,
+            int tier,
+            ClusterNode node,
+            Map<Integer, Queue<Integer>> pendingParts, boolean force,
+            boolean allowNeighbors, boolean ignoreFilter)
         {
             UUID nodeId = node.id();
 
-            if (isAssignable(part, tier, node, allowNeighbors)) {
+            if (isAssignable(part, tier, node, allowNeighbors, ignoreFilter)) {
                 tierMaps[tier].get(nodeId).add(part);
 
                 fullMap.get(nodeId).add(part);
@@ -929,15 +992,17 @@ public Map<UUID, PartitionSet> tierMapping(int tier) {
          * @param tier Tier.
          * @param node Node.
          * @param allowNeighbors Allow neighbors.
+         * @param ignoreFilter ignore backup filter
          * @return {@code true} if the partition is assignable to the node.
          */
-        private boolean isAssignable(int part, int tier, final ClusterNode node, boolean allowNeighbors) {
+        private boolean isAssignable(int part, int tier, final ClusterNode node, boolean allowNeighbors,
+            boolean ignoreFilter) {
             if (containsPartition(part, node))
                 return false;
 
             if (exclNeighbors)
                 return allowNeighbors || !neighborsContainPartition(node, part);
-            else if (affinityBackupFilter != null) {
+            else if (!ignoreFilter && affinityBackupFilter != null) {
                 List<ClusterNode> assignment = assignments.get(part);
 
                 if (assignment.isEmpty())
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionRechecksBackupFilterTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionRechecksBackupFilterTest.java
new file mode 100644
index 0000000000000..529d41c213bb2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionRechecksBackupFilterTest.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.affinity;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.fair.FairAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * Tests backup filter for {@link FairAffinityFunction}.
+ */
+public abstract class AbstractAffinityFunctionRechecksBackupFilterTest extends GridCommonAbstractTest {
+
+    private static final String ATTRIBUTE_NAME = "machine";
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+    private String attrVal = null;
+
+    protected MyFilter filter = new MyFilter();
+
+    /**
+     * @return Affinity function for affinityFunctionRecheckBackupFilterTest.
+     */
+    protected abstract AffinityFunction affinityFunctionWithAffinityBackupFilter();
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        CacheConfiguration cacheCfg = defaultCacheConfiguration();
+
+        cacheCfg.setCacheMode(PARTITIONED);
+        cacheCfg.setBackups(1);
+
+        cacheCfg.setAffinity(affinityFunctionWithAffinityBackupFilter());
+
+        TcpDiscoverySpi spi = new TcpDiscoverySpi();
+        spi.setIpFinder(IP_FINDER);
+
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCacheConfiguration(cacheCfg);
+        cfg.setDiscoverySpi(spi);
+        cfg.setUserAttributes(F.asMap(ATTRIBUTE_NAME, attrVal));
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception
+     */
+    public void testAffinityFunctionAlwaysKeepBackupCountSave() throws Exception {
+        try {
+            attrVal = "A";
+
+            IgniteEx ignite = startGrid(0);
+            startGrid(1);
+
+            awaitPartitionMapExchange();
+
+            checkBackupsAssigned(ignite);
+
+            attrVal = "B";
+            startGrid(2);
+
+            attrVal = "C";
+            startGrid(3);
+
+            awaitPartitionMapExchange();
+
+            checkAffinity(ignite);
+
+            stopGrid(3);
+
+            stopGrid(2);
+
+            awaitPartitionMapExchange();
+
+            checkBackupsAssigned(ignite);
+
+        }
+        finally {
+            Ignition.stopAll(true);
+        }
+    }
+
+    /**
+     * @throws Exception
+     */
+    public void testAffinityFunctionRechecksAffinityBackupFilter() throws Exception {
+        try {
+            attrVal = "A";
+
+            IgniteEx ignite = startGrid(0);
+            startGrid(1);
+
+            attrVal = "B";
+            startGrid(2);
+
+            attrVal = "C";
+            startGrid(3);
+
+            checkAffinity(ignite);
+        }
+        finally {
+            Ignition.stopAll(true);
+        }
+    }
+
+    /**
+     * @param ignite
+     */
+    private void checkBackupsAssigned(IgniteEx ignite) {
+        Affinity<?> aff = ignite.affinity(null);
+
+        int backups = ignite.cache(null).getConfiguration(CacheConfiguration.class).getBackups();
+
+        for (int i = 0; i < aff.partitions(); i++) {
+            Collection<ClusterNode> nodes = aff.mapPartitionToPrimaryAndBackups(i);
+
+            assertTrue(nodes.size() == backups + 1);
+        }
+    }
+
+    private static void checkAffinity(Ignite ignite) throws InterruptedException {
+        Affinity<?> aff = ignite.affinity(null);
+
+        for (int i = 0; i < aff.partitions(); i++) {
+            Collection<ClusterNode> nodes = aff.mapPartitionToPrimaryAndBackups(i);
+
+            Collection<String> machines = new HashSet<>();
+
+            for (ClusterNode node : nodes)
+                machines.add(node.<String>attribute(ATTRIBUTE_NAME));
+
+            assertTrue(machines.size() > 1);
+        }
+    }
+
+    private static class MyFilter implements IgniteBiPredicate<ClusterNode, List<ClusterNode>> {
+        @Override public boolean apply(ClusterNode clusterNode, List<ClusterNode> clusterNodes) {
+            String machineId = clusterNode.attribute(ATTRIBUTE_NAME);
+
+            for (ClusterNode node : clusterNodes) {
+                if (machineId.equals(node.attribute(ATTRIBUTE_NAME)))
+                    return false;
+            }
+
+            return true;
+        }
+    }
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunctionBackupFilterRecheckTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunctionBackupFilterRecheckTest.java
new file mode 100644
index 0000000000000..7017ff9c30639
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunctionBackupFilterRecheckTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.affinity.fair;
+
+import org.apache.ignite.cache.affinity.AbstractAffinityFunctionRechecksBackupFilterTest;
+import org.apache.ignite.cache.affinity.AffinityFunction;
+
+/**
+ * Tests backup filter for {@link FairAffinityFunction}.
+ */
+public class FairAffinityFunctionBackupFilterRecheckTest extends AbstractAffinityFunctionRechecksBackupFilterTest {
+
+    @Override protected AffinityFunction affinityFunctionWithAffinityBackupFilter() {
+        FairAffinityFunction func = new FairAffinityFunction(8);
+        func.setAffinityBackupFilter(filter);
+
+        return func;
+    }
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index e2ab118bbc88a..f5797467a7e8e 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.cache.affinity.AffinityClientNodeSelfTest;
 import org.apache.ignite.cache.affinity.AffinityHistoryCleanupTest;
 import org.apache.ignite.cache.affinity.fair.FairAffinityDynamicCacheSelfTest;
+import org.apache.ignite.cache.affinity.fair.FairAffinityFunctionBackupFilterRecheckTest;
 import org.apache.ignite.cache.affinity.fair.FairAffinityFunctionNodesSelfTest;
 import org.apache.ignite.cache.affinity.fair.FairAffinityFunctionSelfTest;
 import org.apache.ignite.cache.affinity.fair.FairAffinityNodesRestart;
@@ -245,6 +246,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(AffinityClientNodeSelfTest.class);
         suite.addTestSuite(LocalAffinityFunctionTest.class);
         suite.addTestSuite(AffinityHistoryCleanupTest.class);
+        suite.addTestSuite(FairAffinityFunctionBackupFilterRecheckTest.class);
 
         // Swap tests.
         suite.addTestSuite(GridCacheSwapPreloadSelfTest.class);

From 8caa5276794105a3cc2c3988d47beafd6d2c0b5e Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Sun, 21 Jan 2018 21:23:01 +0300
Subject: [PATCH 476/516] IGNITE-7284: Introduce DEV_ONLY marker to
 IgniteLogger.

Signed-off-by: Valentin Kulichenko <valentin.lulichenko@gmail.com>

(cherry picked from commit edc5a5f)
---
 config/ignite-log4j2.xml                      |   6 +-
 .../java/org/apache/ignite/IgniteLogger.java  |  73 +++++++++---
 .../apache/ignite/IgniteSystemProperties.java |   6 +
 .../ignite/internal/GridLoggerProxy.java      |  25 ++++
 .../binary/BinaryClassDescriptor.java         |   5 +-
 .../logger/platform/PlatformLogger.java       |  23 ++++
 .../processors/cache/GridCacheLogger.java     |  25 ++++
 .../ignite/internal/util/IgniteUtils.java     |  24 ++++
 .../org/apache/ignite/logger/NullLogger.java  |  25 ++++
 .../apache/ignite/logger/java/JavaLogger.java |  25 ++++
 modules/core/src/test/config/log4j2-test.xml  |   6 +-
 .../src/test/config/log4j2-verbose-test.xml   |   4 +-
 .../internal/util/IgniteDevOnlyLogTest.java   | 105 +++++++++++++++++
 .../testframework/GridStringLogger.java       |  25 ++++
 .../junits/logger/GridTestLog4jLogger.java    |  25 ++++
 .../hadoop/impl/igfs/HadoopIgfsJclLogger.java |  25 ++++
 .../apache/ignite/logger/jcl/JclLogger.java   |  25 ++++
 .../ignite/logger/log4j/Log4JLogger.java      |  26 ++++-
 modules/log4j2/pom.xml                        |   4 +-
 .../ignite/logger/log4j2/Log4J2Logger.java    |  76 +++++++++---
 .../log4j2/src/test/config/log4j2-markers.xml |  38 ++++++
 .../logger/log4j2/Log4j2LoggerMarkerTest.java | 110 ++++++++++++++++++
 .../testsuites/IgniteLog4j2TestSuite.java     |   4 +-
 modules/slf4j/pom.xml                         |  22 ++++
 .../ignite/logger/slf4j/Slf4jLogger.java      |  46 ++++++--
 .../logger/slf4j/Slf4jLoggerMarkerTest.java   | 108 +++++++++++++++++
 .../testsuites/IgniteSlf4jTestSuite.java      |  37 ++++++
 .../slf4j/src/test/resources/log4j2-test.xml  |  38 ++++++
 parent/pom.xml                                |   1 +
 29 files changed, 912 insertions(+), 50 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/IgniteDevOnlyLogTest.java
 create mode 100644 modules/log4j2/src/test/config/log4j2-markers.xml
 create mode 100644 modules/log4j2/src/test/java/org/apache/ignite/logger/log4j2/Log4j2LoggerMarkerTest.java
 create mode 100644 modules/slf4j/src/test/java/org/apache/ignite/logger/slf4j/Slf4jLoggerMarkerTest.java
 create mode 100644 modules/slf4j/src/test/java/org/apache/ignite/testsuites/IgniteSlf4jTestSuite.java
 create mode 100644 modules/slf4j/src/test/resources/log4j2-test.xml

diff --git a/config/ignite-log4j2.xml b/config/ignite-log4j2.xml
index c9c349e14eb00..7154ae8ca0a1e 100644
--- a/config/ignite-log4j2.xml
+++ b/config/ignite-log4j2.xml
@@ -20,12 +20,12 @@
 <Configuration>
     <Appenders>
         <Console name="CONSOLE" target="SYSTEM_OUT">
-            <PatternLayout pattern="[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n"/>
+            <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
             <ThresholdFilter level="ERROR" onMatch="DENY" onMismatch="ACCEPT"/>
         </Console>
 
         <Console name="CONSOLE_ERR" target="SYSTEM_ERR">
-            <PatternLayout pattern="[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n"/>
+            <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
         </Console>
 
         <Routing name="FILE">
@@ -33,7 +33,7 @@
                 <Route>
                     <RollingFile name="Rolling-${sys:nodeId}" fileName="${sys:IGNITE_HOME}/work/log/ignite-${sys:nodeId}.log"
                                  filePattern="${sys:IGNITE_HOME}/work/log/ignite-${sys:nodeId}-%i-%d{yyyy-MM-dd}.log.gz">
-                        <PatternLayout pattern="[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n"/>
+                        <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
                         <Policies>
                             <TimeBasedTriggeringPolicy interval="6" modulate="true" />
                             <SizeBasedTriggeringPolicy size="10 MB" />
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java b/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
index b1433a8ac230a..b6c7d4debffd2 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
@@ -65,41 +65,70 @@
  */
 @GridToStringExclude
 public interface IgniteLogger {
+    /**
+     * Marker for log messages that are useful in development environments, but not in production.
+     */
+    String DEV_ONLY = "DEV_ONLY";
+
     /**
      * Creates new logger with given category based off the current instance.
      *
      * @param ctgr Category for new logger.
      * @return New logger with given category.
      */
-    public IgniteLogger getLogger(Object ctgr);
+    IgniteLogger getLogger(Object ctgr);
+
+    /**
+     * Logs out trace message.
+     *
+     * @param msg Trace message.
+     */
+    void trace(String msg);
 
     /**
      * Logs out trace message.
      *
+     * @param marker Name of the marker to be associated with the message.
      * @param msg Trace message.
      */
-    public void trace(String msg);
+    void trace(String marker, String msg);
 
     /**
      * Logs out debug message.
      *
      * @param msg Debug message.
      */
-    public void debug(String msg);
+    void debug(String msg);
+
+    /**
+     * Logs out debug message.
+     *
+     * @param marker Name of the marker to be associated with the message.
+     * @param msg Debug message.
+     */
+    void debug(String marker, String msg);
 
     /**
      * Logs out information message.
      *
      * @param msg Information message.
      */
-    public void info(String msg);
+    void info(String msg);
+
+    /**
+     * Logs out information message.
+     *
+     * @param marker Name of the marker to be associated with the message.
+     * @param msg Information message.
+     */
+    void info(String marker, String msg);
 
     /**
      * Logs out warning message.
      *
      * @param msg Warning message.
      */
-    public void warning(String msg);
+    void warning(String msg);
 
     /**
      * Logs out warning message with optional exception.
@@ -107,55 +136,73 @@ public interface IgniteLogger {
      * @param msg Warning message.
      * @param e Optional exception (can be {@code null}).
      */
-    public void warning(String msg, @Nullable Throwable e);
+    void warning(String msg, @Nullable Throwable e);
+
+    /**
+     * Logs out warning message with optional exception.
+     *
+     * @param marker Name of the marker to be associated with the message.
+     * @param msg Warning message.
+     * @param e Optional exception (can be {@code null}).
+     */
+    void warning(String marker, String msg, @Nullable Throwable e);
 
     /**
      * Logs out error message.
      *
      * @param msg Error message.
      */
-    public void error(String msg);
+    void error(String msg);
+
+    /**
+     * Logs error message with optional exception.
+     *
+     * @param msg Error message.
+     * @param e Optional exception (can be {@code null}).
+     */
+    void error(String msg, @Nullable Throwable e);
 
     /**
      * Logs error message with optional exception.
      *
+     * @param marker Name of the marker to be associated with the message.
      * @param msg Error message.
      * @param e Optional exception (can be {@code null}).
      */
-    public void error(String msg, @Nullable Throwable e);
+    void error(String marker, String msg, @Nullable Throwable e);
 
     /**
      * Tests whether {@code trace} level is enabled.
      *
      * @return {@code true} in case when {@code trace} level is enabled, {@code false} otherwise.
      */
-    public boolean isTraceEnabled();
+    boolean isTraceEnabled();
 
     /**
      * Tests whether {@code debug} level is enabled.
      *
      * @return {@code true} in case when {@code debug} level is enabled, {@code false} otherwise.
      */
-    public boolean isDebugEnabled();
+    boolean isDebugEnabled();
 
     /**
      * Tests whether {@code info} level is enabled.
      *
      * @return {@code true} in case when {@code info} level is enabled, {@code false} otherwise.
      */
-    public boolean isInfoEnabled();
+    boolean isInfoEnabled();
 
     /**
      * Tests whether Logger is in "Quiet mode".
      *
      * @return {@code true} "Quiet mode" is enabled, {@code false} otherwise
      */
-    public boolean isQuiet();
+    boolean isQuiet();
 
     /**
      * Gets name of the file being logged to if one is configured or {@code null} otherwise.
      *
      * @return Name of the file being logged to if one is configured or {@code null} otherwise.
      */
-    public String fileName();
+    String fileName();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 16ae83a1ee6d3..9d830dd908606 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -610,6 +610,12 @@ public final class IgniteSystemProperties {
      * that means cache is invalid.*/
     public static final String IGNITE_CACHE_VALIDATOR = "IGNITE_CACHE_VALIDATOR";
 
+    /**
+     * When set to {@code true}, warnings that are intended for development environments and not for production
+     * (such as coding mistakes in code using Ignite) will not be logged.
+     */
+    public static final String IGNITE_DEV_ONLY_LOGGING_DISABLED = "IGNITE_DEV_ONLY_LOGGING_DISABLED";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridLoggerProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/GridLoggerProxy.java
index b26921bbd8d95..655a72ada67f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridLoggerProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridLoggerProxy.java
@@ -116,16 +116,31 @@ public GridLoggerProxy(IgniteLogger impl, @Nullable Object ctgr, @Nullable Strin
         impl.trace(enrich(msg));
     }
 
+    /** {@inheritDoc} */
+    @Override public void trace(@Nullable String marker, String msg) {
+        impl.trace(marker, enrich(msg));
+    }
+
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
         impl.debug(enrich(msg));
     }
 
+    /** {@inheritDoc} */
+    @Override public void debug(@Nullable String marker, String msg) {
+        impl.debug(marker, enrich(msg));
+    }
+
     /** {@inheritDoc} */
     @Override public void info(String msg) {
         impl.info(enrich(msg));
     }
 
+    /** {@inheritDoc} */
+    @Override public void info(@Nullable String marker, String msg) {
+        impl.info(marker, enrich(msg));
+    }
+
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
         impl.warning(enrich(msg));
@@ -136,6 +151,11 @@ public GridLoggerProxy(IgniteLogger impl, @Nullable Object ctgr, @Nullable Strin
         impl.warning(enrich(msg), e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void warning(@Nullable String marker, String msg, @Nullable Throwable e) {
+        impl.warning(marker, enrich(msg), e);
+    }
+
     /** {@inheritDoc} */
     @Override public void error(String msg) {
         impl.error(enrich(msg));
@@ -146,6 +166,11 @@ public GridLoggerProxy(IgniteLogger impl, @Nullable Object ctgr, @Nullable Strin
         impl.error(enrich(msg), e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void error(@Nullable String marker, String msg, @Nullable Throwable e) {
+        impl.error(marker, enrich(msg), e);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return impl.isTraceEnabled();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
index 3a72a31db58aa..dd69bfb321e90 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
@@ -189,9 +189,8 @@ else if (useOptMarshaller)
                 mode = serializer != null ? BinaryWriteMode.BINARY : BinaryUtils.mode(cls);
         }
 
-        if (useOptMarshaller && userType && !U.isIgnite(cls) && !U.isJdk(cls) &&
-            !GridQueryProcessor.isGeometryClass(cls)) {
-            U.warn(ctx.log(), "Class \"" + cls.getName() + "\" cannot be serialized using " +
+        if (useOptMarshaller && userType && !U.isIgnite(cls) && !U.isJdk(cls) && !GridQueryProcessor.isGeometryClass(cls)) {
+            U.warnDevOnly(ctx.log(), "Class \"" + cls.getName() + "\" cannot be serialized using " +
                 BinaryMarshaller.class.getSimpleName() + " because it either implements Externalizable interface " +
                 "or have writeObject/readObject methods. " + OptimizedMarshaller.class.getSimpleName() + " will be " +
                 "used instead and class instances will be deserialized on the server. Please ensure that all nodes " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/logger/platform/PlatformLogger.java b/modules/core/src/main/java/org/apache/ignite/internal/logger/platform/PlatformLogger.java
index 0a0437e9eed89..60e50d8a41c5d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/logger/platform/PlatformLogger.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/logger/platform/PlatformLogger.java
@@ -100,16 +100,30 @@ private PlatformLogger(PlatformCallbackGateway gate, PlatformContext ctx, String
         log(LVL_TRACE, msg, null);
     }
 
+    /** {@inheritDoc} */
+    @Override public void trace(String marker, String msg) {
+        trace(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
         log(LVL_DEBUG, msg, null);
     }
 
+    @Override public void debug(String marker, String msg) {
+        debug(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void info(String msg) {
         log(LVL_INFO, msg, null);
     }
 
+    /** {@inheritDoc} */
+    @Override public void info(String marker, String msg) {
+        info(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
         log(LVL_WARN, msg, null);
@@ -120,6 +134,11 @@ private PlatformLogger(PlatformCallbackGateway gate, PlatformContext ctx, String
         log(LVL_WARN, msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void warning(String marker, String msg, @Nullable Throwable e) {
+        warning(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public void error(String msg) {
         log(LVL_ERROR, msg, null);
@@ -130,6 +149,10 @@ private PlatformLogger(PlatformCallbackGateway gate, PlatformContext ctx, String
         log(LVL_ERROR, msg, e);
     }
 
+    @Override public void error(String marker, String msg, @Nullable Throwable e) {
+        error(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return traceEnabled;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLogger.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLogger.java
index 7348c67f715e7..8ab0c52540a43 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLogger.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLogger.java
@@ -94,6 +94,11 @@ private String format(String msg) {
         log.debug(format(msg));
     }
 
+    /** {@inheritDoc} */
+    @Override public void debug(String marker, String msg) {
+        log.debug(marker, msg);
+    }
+
     /** {@inheritDoc} */
     @Override public IgniteLogger getLogger(Object ctgr) {
         return new GridCacheLogger(cctx, ctgr.toString());
@@ -104,11 +109,21 @@ private String format(String msg) {
         log.trace(format(msg));
     }
 
+    /** {@inheritDoc} */
+    @Override public void trace(String marker, String msg) {
+        log.trace(marker, msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void info(String msg) {
         log.info(format(msg));
     }
 
+    /** {@inheritDoc} */
+    @Override public void info(String marker, String msg) {
+        log.info(marker, msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
         log.warning(format(msg));
@@ -119,6 +134,11 @@ private String format(String msg) {
         log.warning(format(msg), e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void warning(String marker, String msg, @Nullable Throwable e) {
+        log.warning(marker, msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public void error(String msg) {
         log.error(format(msg));
@@ -129,6 +149,11 @@ private String format(String msg) {
         log.error(format(msg), e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void error(String marker, String msg, @Nullable Throwable e) {
+        log.error(marker, msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return log.isTraceEnabled();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 79f91663e6334..73055ae8bdab4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -4221,6 +4221,30 @@ public static void warn(@Nullable IgniteLogger log, Object longMsg, Object short
                 compact(shortMsg.toString()));
     }
 
+    /**
+     * Depending on whether or not log is provided and quiet mode is enabled logs given
+     * messages as quiet message or normal log WARN message with {@link IgniteLogger#DEV_ONLY DEV_ONLY} marker.
+     * If {@code log} is {@code null} or in QUIET mode it will add {@code (wrn)} prefix to the message.
+     * If property {@link IgniteSystemProperties#IGNITE_DEV_ONLY_LOGGING_DISABLED IGNITE_DEV_ONLY_LOGGING_DISABLED}
+     * is set to true, the message will not be logged.
+     *
+     * @param log Optional logger to use when QUIET mode is not enabled.
+     * @param msg Message to log.
+     */
+    public static void warnDevOnly(@Nullable IgniteLogger log, Object msg) {
+        assert msg != null;
+
+        // don't log message if DEV_ONLY messages are disabled
+        if (IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_DEV_ONLY_LOGGING_DISABLED))
+            return;
+
+        if (log != null)
+            log.warning(IgniteLogger.DEV_ONLY, compact(msg.toString()), null);
+        else
+            X.println("[" + SHORT_DATE_FMT.format(new java.util.Date()) + "] (wrn) " +
+                compact(msg.toString()));
+    }
+
     /**
      * Depending on whether or not log is provided and quiet mode is enabled logs given
      * messages as quiet message or normal log INFO message.
diff --git a/modules/core/src/main/java/org/apache/ignite/logger/NullLogger.java b/modules/core/src/main/java/org/apache/ignite/logger/NullLogger.java
index 71c7ba499d1ff..c935fea13056c 100644
--- a/modules/core/src/main/java/org/apache/ignite/logger/NullLogger.java
+++ b/modules/core/src/main/java/org/apache/ignite/logger/NullLogger.java
@@ -34,16 +34,31 @@ public class NullLogger implements IgniteLogger {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void trace(String marker, String msg) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void debug(String marker, String msg) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void info(String msg) {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void info(String marker, String msg) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
         // No-op.
@@ -54,6 +69,11 @@ public class NullLogger implements IgniteLogger {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void warning(String marker, String msg, @Nullable Throwable e) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void error(String msg) {
         // No-op.
@@ -64,6 +84,11 @@ public class NullLogger implements IgniteLogger {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void error(String marker, String msg, @Nullable Throwable e) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/logger/java/JavaLogger.java b/modules/core/src/main/java/org/apache/ignite/logger/java/JavaLogger.java
index 6aa7d3879698a..c03592a5b1b59 100644
--- a/modules/core/src/main/java/org/apache/ignite/logger/java/JavaLogger.java
+++ b/modules/core/src/main/java/org/apache/ignite/logger/java/JavaLogger.java
@@ -257,6 +257,11 @@ private void configure(@Nullable Logger initImpl) {
         impl.finest(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void trace(@Nullable String marker, String msg) {
+        trace(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
         if (!impl.isLoggable(FINE))
@@ -265,6 +270,11 @@ private void configure(@Nullable Logger initImpl) {
         impl.fine(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void debug(@Nullable String marker, String msg) {
+        debug(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void info(String msg) {
         if (!impl.isLoggable(INFO))
@@ -273,6 +283,11 @@ private void configure(@Nullable Logger initImpl) {
         impl.info(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void info(@Nullable String marker, String msg) {
+        info(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
         impl.warning(msg);
@@ -283,6 +298,11 @@ private void configure(@Nullable Logger initImpl) {
         impl.log(WARNING, msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void warning(@Nullable String marker, String msg, @Nullable Throwable e) {
+        warning(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public void error(String msg) {
         impl.severe(msg);
@@ -298,6 +318,11 @@ private void configure(@Nullable Logger initImpl) {
         impl.log(SEVERE, msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void error(@Nullable String marker, String msg, @Nullable Throwable e) {
+        error(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return impl.isLoggable(FINEST);
diff --git a/modules/core/src/test/config/log4j2-test.xml b/modules/core/src/test/config/log4j2-test.xml
index 30599cc8fdeae..8b46f00bca50e 100644
--- a/modules/core/src/test/config/log4j2-test.xml
+++ b/modules/core/src/test/config/log4j2-test.xml
@@ -20,12 +20,12 @@
 <Configuration>
     <Appenders>
         <Console name="CONSOLE" target="SYSTEM_OUT">
-            <PatternLayout pattern="[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n"/>
+            <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
             <ThresholdFilter level="ERROR" onMatch="DENY" onMismatch="ACCEPT"/>
         </Console>
 
         <Console name="CONSOLE_ERR" target="SYSTEM_ERR">
-            <PatternLayout pattern="[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n"/>
+            <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
         </Console>
 
         <Routing name="FILE">
@@ -33,7 +33,7 @@
                 <Route>
                     <RollingFile name="Rolling-${sys:nodeId}" fileName="${sys:IGNITE_HOME}/work/log/ignite-${sys:nodeId}.log"
                                  filePattern="${sys:IGNITE_HOME}/work/log/ignite-${sys:nodeId}-%i-%d{yyyy-MM-dd}.log.gz">
-                        <PatternLayout pattern="[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n"/>
+                        <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
                         <Policies>
                             <TimeBasedTriggeringPolicy interval="6" modulate="true" />
                             <SizeBasedTriggeringPolicy size="10 MB" />
diff --git a/modules/core/src/test/config/log4j2-verbose-test.xml b/modules/core/src/test/config/log4j2-verbose-test.xml
index b870dc824d7c3..ff6325b290513 100644
--- a/modules/core/src/test/config/log4j2-verbose-test.xml
+++ b/modules/core/src/test/config/log4j2-verbose-test.xml
@@ -20,7 +20,7 @@
 <Configuration>
     <Appenders>
         <Console name="CONSOLE_ERR" target="SYSTEM_ERR">
-            <PatternLayout pattern="[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n"/>
+            <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
         </Console>
 
         <Routing name="FILE">
@@ -28,7 +28,7 @@
                 <Route>
                     <RollingFile name="Rolling-${sys:nodeId}" fileName="${sys:IGNITE_HOME}/work/log/ignite-${sys:nodeId}.log"
                                  filePattern="${sys:IGNITE_HOME}/work/log/ignite-${sys:nodeId}-%i-%d{yyyy-MM-dd}.log.gz">
-                        <PatternLayout pattern="[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n"/>
+                        <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
                         <Policies>
                             <TimeBasedTriggeringPolicy interval="6" modulate="true" />
                             <SizeBasedTriggeringPolicy size="10 MB" />
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteDevOnlyLogTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteDevOnlyLogTest.java
new file mode 100644
index 0000000000000..87b44da7f2cfe
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteDevOnlyLogTest.java
@@ -0,0 +1,105 @@
+/* @java.file.header */
+
+/*  _________        _____ __________________        _____
+ *  __  ____/___________(_)______  /__  ____/______ ____(_)_______
+ *  _  / __  __  ___/__  / _  __  / _  / __  _  __ `/__  / __  __ \
+ *  / /_/ /  _  /    _  /  / /_/ /  / /_/ /  / /_/ / _  /  _  / / /
+ *  \____/   /_/     /_/   \_,__/   \____/   \__,_/  /_/   /_/ /_/
+ */
+
+package org.apache.ignite.internal.util;
+
+import java.io.IOException;
+import java.util.Collections;
+import junit.framework.TestCase;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+
+/**
+ * Testing logging via {@link IgniteUtils#warnDevOnly(IgniteLogger, Object)}.
+ */
+public class IgniteDevOnlyLogTest extends TestCase {
+    /** Check that dev-only messages appear in the log. */
+    public void testDevOnlyQuietMessage() throws IOException {
+        String oldQuietVal = System.setProperty(IgniteSystemProperties.IGNITE_QUIET, "true");
+
+        try (Ignite ignite = startNode()) {
+            String msg = getMessage(ignite);
+            IgniteUtils.warnDevOnly(ignite.log(), msg);
+            assertTrue(readLog(ignite).contains(msg));
+        }
+        finally {
+            setOrClearProperty(IgniteSystemProperties.IGNITE_QUIET, oldQuietVal);
+        }
+    }
+
+    /** Check that dev-only messages appear in the log. */
+    public void testDevOnlyVerboseMessage() throws IOException {
+        String oldQuietVal = System.setProperty(IgniteSystemProperties.IGNITE_QUIET, "false");
+
+        try (Ignite ignite = startNode()) {
+            String msg = getMessage(ignite);
+            IgniteUtils.warnDevOnly(ignite.log(), msg);
+            assertTrue(readLog(ignite).contains(msg));
+        }
+        finally {
+            setOrClearProperty(IgniteSystemProperties.IGNITE_QUIET, oldQuietVal);
+        }
+    }
+
+    /**
+     * Check that {@link IgniteUtils#warnDevOnly(IgniteLogger, Object)}
+     * doesn't print anything if {@link org.apache.ignite.IgniteSystemProperties#IGNITE_DEV_ONLY_LOGGING_DISABLED}
+     * is set to {@code true}.
+     */
+    public void testDevOnlyDisabledProperty() throws IOException {
+        String oldDevOnlyVal = System.setProperty(IgniteSystemProperties.IGNITE_DEV_ONLY_LOGGING_DISABLED, "true");
+
+        try (Ignite ignite = startNode()) {
+            String msg = getMessage(ignite);
+            IgniteUtils.warnDevOnly(ignite.log(), msg);
+            assertFalse(readLog(ignite).contains(msg));
+        }
+        finally {
+            setOrClearProperty(IgniteSystemProperties.IGNITE_DEV_ONLY_LOGGING_DISABLED, oldDevOnlyVal);
+        }
+
+    }
+
+    /** Sets a system property if the value is not null, or clears it if the value is null. */
+    private void setOrClearProperty(String key, String val) {
+        if (val != null)
+            System.setProperty(key, val);
+        else
+            System.clearProperty(IgniteSystemProperties.IGNITE_QUIET);
+    }
+
+    /** Starts an Ignite node. */
+    private Ignite startNode() throws IOException {
+        final TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
+        ipFinder.setAddresses(Collections.singletonList("127.0.0.1:47500..47509"));
+        IgniteConfiguration configuration = new IgniteConfiguration()
+            .setGridName(IgniteDevOnlyLogTest.class.getName() + "Instance")
+            .setDiscoverySpi(new TcpDiscoverySpi()
+                .setIpFinder(ipFinder)
+            );
+
+        return Ignition.start(configuration);
+    }
+
+    /** Reads log of the given node to a string. */
+    private String readLog(Ignite ignite) throws IOException {
+        return IgniteUtils.readFileToString(ignite.log().fileName(), "UTF-8");
+    }
+
+    /** Returns a test message. */
+    private String getMessage(Ignite ignite) {
+        // use node id in the message to avoid interference with other tests
+        return "My id is " + ignite.cluster().localNode().id();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java
index 2a25542b92f1f..da1398e875e77 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java
@@ -92,16 +92,31 @@ private void log(String msg) {
         log(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void trace(String marker, String msg) {
+        trace(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
         log(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void debug(String marker, String msg) {
+        debug(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void info(String msg) {
         log(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void info(String marker, String msg) {
+        info(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
         log(msg);
@@ -115,6 +130,11 @@ private void log(String msg) {
             log(e.toString());
     }
 
+    /** {@inheritDoc} */
+    @Override public void warning(String marker, String msg, @Nullable Throwable e) {
+        warning(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public void error(String msg) {
         log(msg);
@@ -128,6 +148,11 @@ private void log(String msg) {
             log(e.toString());
     }
 
+    /** {@inheritDoc} */
+    @Override public void error(String marker, String msg, @Nullable Throwable e) {
+        error(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return dbg;
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLogger.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLogger.java
index 6a46c7d7cc7da..50d3a2c4c97f9 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLogger.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLogger.java
@@ -461,6 +461,11 @@ public static Collection<String> logFiles() {
         impl.trace(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void trace(String marker, String msg) {
+        trace(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
         if (!impl.isDebugEnabled())
@@ -469,6 +474,11 @@ public static Collection<String> logFiles() {
         impl.debug(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void debug(String marker, String msg) {
+        debug(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void info(String msg) {
         if (!impl.isInfoEnabled())
@@ -477,6 +487,11 @@ public static Collection<String> logFiles() {
         impl.info(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void info(String marker, String msg) {
+        info(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
         impl.warn(msg);
@@ -487,6 +502,11 @@ public static Collection<String> logFiles() {
         impl.warn(msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void warning(String marker, String msg, @Nullable Throwable e) {
+        warning(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public void error(String msg) {
         impl.error(msg);
@@ -497,6 +517,11 @@ public static Collection<String> logFiles() {
         impl.error(msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void error(String marker, String msg, @Nullable Throwable e) {
+        error(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return impl.isTraceEnabled();
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsJclLogger.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsJclLogger.java
index 64752040fe446..b1825c7eb4e10 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsJclLogger.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsJclLogger.java
@@ -61,16 +61,31 @@ public class HadoopIgfsJclLogger implements IgniteLogger {
         impl.trace(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void trace(String marker, String msg) {
+        trace(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
         impl.debug(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void debug(String marker, String msg) {
+        debug(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void info(String msg) {
         impl.info(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void info(String marker, String msg) {
+        info(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
         impl.warn(msg);
@@ -81,6 +96,11 @@ public class HadoopIgfsJclLogger implements IgniteLogger {
         impl.warn(msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void warning(String marker, String msg, @Nullable Throwable e) {
+        warning(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public void error(String msg) {
         impl.error(msg);
@@ -96,6 +116,11 @@ public class HadoopIgfsJclLogger implements IgniteLogger {
         impl.error(msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void error(String marker, String msg, @Nullable Throwable e) {
+        error(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return impl.isTraceEnabled();
diff --git a/modules/jcl/src/main/java/org/apache/ignite/logger/jcl/JclLogger.java b/modules/jcl/src/main/java/org/apache/ignite/logger/jcl/JclLogger.java
index c75197a02977f..0430da03f17a1 100644
--- a/modules/jcl/src/main/java/org/apache/ignite/logger/jcl/JclLogger.java
+++ b/modules/jcl/src/main/java/org/apache/ignite/logger/jcl/JclLogger.java
@@ -113,16 +113,31 @@ public JclLogger(Log impl) {
         impl.trace(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void trace(String marker, String msg) {
+        trace(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
         impl.debug(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void debug(String marker, String msg) {
+        debug(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void info(String msg) {
         impl.info(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void info(String marker, String msg) {
+        info(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
         impl.warn(msg);
@@ -133,6 +148,11 @@ public JclLogger(Log impl) {
         impl.warn(msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void warning(String marker, String msg, @Nullable Throwable e) {
+        warning(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public void error(String msg) {
         impl.error(msg);
@@ -148,6 +168,11 @@ public JclLogger(Log impl) {
         impl.error(msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void error(String marker, String msg, @Nullable Throwable e) {
+        error(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return impl.isTraceEnabled();
diff --git a/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java b/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
index a49e24716c399..568d6da19ce9d 100644
--- a/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
+++ b/modules/log4j/src/main/java/org/apache/ignite/logger/log4j/Log4JLogger.java
@@ -35,7 +35,6 @@
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.logger.LoggerNodeIdAware;
 import org.apache.log4j.Appender;
-import org.apache.log4j.AppenderSkeleton;
 import org.apache.log4j.Category;
 import org.apache.log4j.ConsoleAppender;
 import org.apache.log4j.FileAppender;
@@ -491,6 +490,11 @@ public static Collection<String> logFiles() {
         impl.trace(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void trace(String marker, String msg) {
+        trace(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
         if (!impl.isDebugEnabled())
@@ -499,6 +503,11 @@ public static Collection<String> logFiles() {
         impl.debug(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void debug(String marker, String msg) {
+        debug(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void info(String msg) {
         if (!impl.isInfoEnabled())
@@ -507,6 +516,11 @@ public static Collection<String> logFiles() {
         impl.info(msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void info(String marker, String msg) {
+        info(msg);
+    }
+
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
         impl.warn(msg);
@@ -517,6 +531,11 @@ public static Collection<String> logFiles() {
         impl.warn(msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void warning(String marker, String msg, @Nullable Throwable e) {
+        warning(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public void error(String msg) {
         impl.error(msg);
@@ -527,6 +546,11 @@ public static Collection<String> logFiles() {
         impl.error(msg, e);
     }
 
+    /** {@inheritDoc} */
+    @Override public void error(String marker, String msg, @Nullable Throwable e) {
+        error(msg, e);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return impl.isTraceEnabled();
diff --git a/modules/log4j2/pom.xml b/modules/log4j2/pom.xml
index 5b0e9897cda0f..f880c1075d808 100644
--- a/modules/log4j2/pom.xml
+++ b/modules/log4j2/pom.xml
@@ -52,13 +52,13 @@
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-api</artifactId>
-            <version>2.3</version>
+            <version>${log4j.version}</version>
         </dependency>
 
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-core</artifactId>
-            <version>2.3</version>
+            <version>${log4j.version}</version>
         </dependency>
     </dependencies>
 
diff --git a/modules/log4j2/src/main/java/org/apache/ignite/logger/log4j2/Log4J2Logger.java b/modules/log4j2/src/main/java/org/apache/ignite/logger/log4j2/Log4J2Logger.java
index ffe8e1b6043a1..ec72db1691eac 100644
--- a/modules/log4j2/src/main/java/org/apache/ignite/logger/log4j2/Log4J2Logger.java
+++ b/modules/log4j2/src/main/java/org/apache/ignite/logger/log4j2/Log4J2Logger.java
@@ -34,6 +34,8 @@
 import org.apache.ignite.logger.LoggerNodeIdAware;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.MarkerManager;
 import org.apache.logging.log4j.core.Appender;
 import org.apache.logging.log4j.core.Logger;
 import org.apache.logging.log4j.core.LoggerContext;
@@ -203,6 +205,21 @@ public Log4J2Logger(final URL cfgUrl) throws IgniteCheckedException {
         quiet = quiet0;
     }
 
+
+    /**
+     * Cleans up the logger configuration. Should be used in unit tests only for sequential tests run with
+     * different configurations
+     *
+     */
+    static void cleanup() {
+        synchronized (mux) {
+            if (inited)
+                LogManager.shutdown();
+
+            inited = false;
+        }
+    }
+
     /**
      * Sets level for internal log4j implementation.
      *
@@ -353,8 +370,13 @@ public static Logger createConsoleLogger() {
 
         Configuration cfg = ctx.getConfiguration();
 
-        PatternLayout layout = PatternLayout.createLayout("[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n", null, null,
-            Charset.defaultCharset(), false, false, null, null);
+        PatternLayout.Builder builder = PatternLayout.newBuilder()
+            .withPattern("[%d{ABSOLUTE}][%-5p][%t][%c{1}] %m%n")
+            .withCharset(Charset.defaultCharset())
+            .withAlwaysWriteExceptions(false)
+            .withNoConsoleNoAnsi(false);
+
+        PatternLayout layout = builder.build();
 
         final Appender consoleApp = ConsoleAppender.createAppender(layout, null, null, CONSOLE_APPENDER, null, null);
         consoleApp.start();
@@ -422,10 +444,15 @@ public static Logger createConsoleLogger() {
 
     /** {@inheritDoc} */
     @Override public void trace(String msg) {
+        trace(null, msg);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void trace(@Nullable String marker, String msg) {
         if (!isTraceEnabled())
             warning("Logging at TRACE level without checking if TRACE level is enabled: " + msg);
 
-        impl.trace(msg);
+        impl.trace(getMarkerOrNull(marker), msg);
 
         if (consoleLog != null)
             consoleLog.trace(msg);
@@ -433,10 +460,15 @@ public static Logger createConsoleLogger() {
 
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
+        debug(null, msg);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void debug(@Nullable String marker, String msg) {
         if (!isDebugEnabled())
             warning("Logging at DEBUG level without checking if DEBUG level is enabled: " + msg);
 
-        impl.debug(msg);
+        impl.debug(getMarkerOrNull(marker), msg);
 
         if (consoleLog != null)
             consoleLog.debug(msg);
@@ -444,10 +476,15 @@ public static Logger createConsoleLogger() {
 
     /** {@inheritDoc} */
     @Override public void info(String msg) {
+        info(null, msg);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void info(@Nullable String marker, String msg) {
         if (!isInfoEnabled())
             warning("Logging at INFO level without checking if INFO level is enabled: " + msg);
 
-        impl.info(msg);
+        impl.info(getMarkerOrNull(marker), msg);
 
         if (consoleLog != null)
             consoleLog.info(msg);
@@ -455,15 +492,17 @@ public static Logger createConsoleLogger() {
 
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
-        impl.warn(msg);
-
-        if (consoleLog != null)
-            consoleLog.warn(msg);
+        warning(null, msg, null);
     }
 
     /** {@inheritDoc} */
     @Override public void warning(String msg, @Nullable Throwable e) {
-        impl.warn(msg, e);
+        warning(null, msg, e);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void warning(@Nullable String marker, String msg, @Nullable Throwable e) {
+        impl.warn(getMarkerOrNull(marker), msg, e);
 
         if (consoleLog != null)
             consoleLog.warn(msg, e);
@@ -471,20 +510,27 @@ public static Logger createConsoleLogger() {
 
     /** {@inheritDoc} */
     @Override public void error(String msg) {
-        impl.error(msg);
-
-        if (consoleLog != null)
-            consoleLog.error(msg);
+        error(null, msg, null);
     }
 
     /** {@inheritDoc} */
     @Override public void error(String msg, @Nullable Throwable e) {
-        impl.error(msg, e);
+        error(null, msg, e);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void error(@Nullable String marker, String msg, @Nullable Throwable e) {
+        impl.error(getMarkerOrNull(marker), msg, e);
 
         if (consoleLog != null)
             consoleLog.error(msg, e);
     }
 
+    /** Returns Marker object for the specified name, or null if the name is null */
+    private Marker getMarkerOrNull(@Nullable String marker) {
+        return marker != null ? MarkerManager.getMarker(marker) : null;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isTraceEnabled() {
         return impl.isTraceEnabled() || (consoleLog != null && consoleLog.isTraceEnabled());
diff --git a/modules/log4j2/src/test/config/log4j2-markers.xml b/modules/log4j2/src/test/config/log4j2-markers.xml
new file mode 100644
index 0000000000000..f9c58985b0171
--- /dev/null
+++ b/modules/log4j2/src/test/config/log4j2-markers.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<Configuration>
+    <Appenders>
+        <File name="FILTERED" fileName="${sys:IGNITE_HOME}/work/log/filtered.log" append="false">
+            <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
+            <MarkerFilter marker="IGNORE_ME" onMatch="DENY" onMismatch="NEUTRAL"/>
+        </File>
+
+        <File name="ALL" fileName="${sys:IGNITE_HOME}/work/log/all.log"  append="false">
+            <PatternLayout pattern="[%d{ISO8601}][%-5p]%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
+        </File>
+    </Appenders>
+
+    <Loggers>
+        <Root level="TRACE">
+            <AppenderRef ref="FILTERED"/>
+            <AppenderRef ref="ALL"/>
+        </Root>
+    </Loggers>
+</Configuration>
diff --git a/modules/log4j2/src/test/java/org/apache/ignite/logger/log4j2/Log4j2LoggerMarkerTest.java b/modules/log4j2/src/test/java/org/apache/ignite/logger/log4j2/Log4j2LoggerMarkerTest.java
new file mode 100644
index 0000000000000..672f5d960974b
--- /dev/null
+++ b/modules/log4j2/src/test/java/org/apache/ignite/logger/log4j2/Log4j2LoggerMarkerTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.logger.log4j2;
+
+import java.io.File;
+import junit.framework.TestCase;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Testing that markers are supported by log4j2 implementation.
+ */
+public class Log4j2LoggerMarkerTest extends TestCase {
+    /** Path to log4j configuration. */
+    private static final String LOG_CONFIG = "modules/log4j2/src/test/config/log4j2-markers.xml";
+
+    /** Path to full log. */
+    private static final String LOG_ALL = "work/log/all.log";
+
+    /** Path to filtered log. */
+    private static final String LOG_FILTERED = "work/log/filtered.log";
+
+    /** */
+    @Override protected void setUp() throws Exception {
+        super.setUp();
+
+        Log4J2Logger.cleanup();
+
+        deleteLogs();
+    }
+
+    /** */
+    @Override protected void tearDown() throws Exception {
+        super.tearDown();
+
+        deleteLogs();
+    }
+
+    /** */
+    public void testMarkerFiltering() throws Exception {
+        // create log
+        Log4J2Logger log = new Log4J2Logger(LOG_CONFIG);
+
+        // populate log with messages
+        log.error("IGNORE_ME", "Ignored error", null);
+        log.warning("IGNORE_ME", "Ignored warning", null);
+        log.info("IGNORE_ME", "Ignored info");
+        log.debug("IGNORE_ME", "Ignored debug");
+        log.trace("IGNORE_ME", "Ignored trace");
+
+        log.error("ACCEPT_ME", "Accepted error", null);
+        log.warning("ACCEPT_ME", "Accepted warning", null);
+        log.info("ACCEPT_ME", "Accepted info");
+        log.debug("ACCEPT_ME", "Accepted debug");
+        log.trace("ACCEPT_ME", "Accepted trace");
+
+        // check file with all messages
+        File allFile = U.resolveIgnitePath(LOG_ALL);
+        assertNotNull(allFile);
+        String all = U.readFileToString(allFile.getPath(), "UTF-8");
+
+        assertTrue(all.contains("Ignored error"));
+        assertTrue(all.contains("Ignored warning"));
+        assertTrue(all.contains("Ignored info"));
+        assertTrue(all.contains("Ignored debug"));
+        assertTrue(all.contains("Ignored trace"));
+        assertTrue(all.contains("Accepted error"));
+        assertTrue(all.contains("Accepted warning"));
+        assertTrue(all.contains("Accepted info"));
+        assertTrue(all.contains("Accepted debug"));
+        assertTrue(all.contains("Accepted trace"));
+
+        // check file with one marker filtered out
+        File filteredFile = U.resolveIgnitePath(LOG_FILTERED);
+        assertNotNull(filteredFile);
+        String filtered = U.readFileToString(filteredFile.getPath(), "UTF-8");
+
+        assertFalse(filtered.contains("Ignored error"));
+        assertFalse(filtered.contains("Ignored warning"));
+        assertFalse(filtered.contains("Ignored info"));
+        assertFalse(filtered.contains("Ignored debug"));
+        assertFalse(filtered.contains("Ignored trace"));
+        assertTrue(filtered.contains("Accepted error"));
+        assertTrue(filtered.contains("Accepted warning"));
+        assertTrue(filtered.contains("Accepted info"));
+        assertTrue(filtered.contains("Accepted debug"));
+        assertTrue(filtered.contains("Accepted trace"));
+    }
+
+    /** Delete existing logs, if any */
+    @SuppressWarnings("ResultOfMethodCallIgnored")
+    private void deleteLogs() {
+        new File(LOG_ALL).delete();
+        new File(LOG_FILTERED).delete();
+    }
+}
diff --git a/modules/log4j2/src/test/java/org/apache/ignite/testsuites/IgniteLog4j2TestSuite.java b/modules/log4j2/src/test/java/org/apache/ignite/testsuites/IgniteLog4j2TestSuite.java
index dd2b4b513e324..d06f1417483d1 100644
--- a/modules/log4j2/src/test/java/org/apache/ignite/testsuites/IgniteLog4j2TestSuite.java
+++ b/modules/log4j2/src/test/java/org/apache/ignite/testsuites/IgniteLog4j2TestSuite.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.logger.log4j2.Log4j2LoggerMarkerTest;
 import org.apache.ignite.logger.log4j2.Log4j2LoggerSelfTest;
 
 /**
@@ -31,7 +32,8 @@ public class IgniteLog4j2TestSuite extends TestSuite {
     public static TestSuite suite() throws Exception {
         TestSuite suite = new TestSuite("Log4j2 Logging Test Suite");
 
-        suite.addTest(new TestSuite(Log4j2LoggerSelfTest.class));
+        suite.addTestSuite(Log4j2LoggerSelfTest.class);
+        suite.addTestSuite(Log4j2LoggerMarkerTest.class);
 
         return suite;
     }
diff --git a/modules/slf4j/pom.xml b/modules/slf4j/pom.xml
index a19b46bf613ac..872a42f31a8bf 100644
--- a/modules/slf4j/pom.xml
+++ b/modules/slf4j/pom.xml
@@ -46,6 +46,28 @@
             <artifactId>slf4j-api</artifactId>
             <version>${slf4j.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-core</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+            <version>${log4j.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <version>${log4j.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
diff --git a/modules/slf4j/src/main/java/org/apache/ignite/logger/slf4j/Slf4jLogger.java b/modules/slf4j/src/main/java/org/apache/ignite/logger/slf4j/Slf4jLogger.java
index 2b0e98041db39..a38e90b89e6d7 100644
--- a/modules/slf4j/src/main/java/org/apache/ignite/logger/slf4j/Slf4jLogger.java
+++ b/modules/slf4j/src/main/java/org/apache/ignite/logger/slf4j/Slf4jLogger.java
@@ -21,6 +21,8 @@
 import org.jetbrains.annotations.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_QUIET;
 
@@ -77,46 +79,76 @@ public Slf4jLogger(Logger impl) {
 
     /** {@inheritDoc} */
     @Override public void trace(String msg) {
+        trace(null, msg);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void trace(@Nullable String marker, String msg) {
         if (!impl.isTraceEnabled())
             warning("Logging at TRACE level without checking if TRACE level is enabled: " + msg);
 
-        impl.trace(msg);
+        impl.trace(getMarkerOrNull(marker), msg);
     }
 
     /** {@inheritDoc} */
     @Override public void debug(String msg) {
+        debug(null, msg);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void debug(@Nullable String marker, String msg) {
         if (!impl.isDebugEnabled())
             warning("Logging at DEBUG level without checking if DEBUG level is enabled: " + msg);
 
-        impl.debug(msg);
+        impl.debug(getMarkerOrNull(marker), msg);
     }
 
     /** {@inheritDoc} */
     @Override public void info(String msg) {
+        info(null, msg);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void info(@Nullable String marker, String msg) {
         if (!impl.isInfoEnabled())
             warning("Logging at INFO level without checking if INFO level is enabled: " + msg);
 
-        impl.info(msg);
+        impl.info(getMarkerOrNull(marker), msg);
     }
 
     /** {@inheritDoc} */
     @Override public void warning(String msg) {
-        impl.warn(msg);
+        warning(null, msg, null);
     }
 
     /** {@inheritDoc} */
     @Override public void warning(String msg, @Nullable Throwable e) {
-        impl.warn(msg, e);
+        warning(null, msg, e);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void warning(@Nullable String marker, String msg, @Nullable Throwable e) {
+        impl.warn(getMarkerOrNull(marker), msg, e);
     }
 
     /** {@inheritDoc} */
     @Override public void error(String msg) {
-        impl.error(msg);
+        warning(null, msg, null);
     }
 
     /** {@inheritDoc} */
     @Override public void error(String msg, @Nullable Throwable e) {
-        impl.error(msg, e);
+        error(null, msg, e);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void error(@Nullable String marker, String msg, @Nullable Throwable e) {
+        impl.error(getMarkerOrNull(marker), msg, e);
+    }
+
+    /** Returns Marker object for the specified name, or null if the name is null */
+    private Marker getMarkerOrNull(@Nullable String marker) {
+        return marker != null ? MarkerFactory.getMarker(marker) : null;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/slf4j/src/test/java/org/apache/ignite/logger/slf4j/Slf4jLoggerMarkerTest.java b/modules/slf4j/src/test/java/org/apache/ignite/logger/slf4j/Slf4jLoggerMarkerTest.java
new file mode 100644
index 0000000000000..578e6ba3def3d
--- /dev/null
+++ b/modules/slf4j/src/test/java/org/apache/ignite/logger/slf4j/Slf4jLoggerMarkerTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.logger.slf4j;
+
+import java.io.File;
+import junit.framework.TestCase;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Testing that markers are supported by log4j2 implementation.
+ */
+public class Slf4jLoggerMarkerTest extends TestCase {
+    /** Path to full log. */
+    private static final String LOG_ALL = "work/log/all.log";
+
+    /** Path to filtered log. */
+    private static final String LOG_FILTERED = "work/log/filtered.log";
+
+    /** */
+    @Override protected void setUp() throws Exception {
+        super.setUp();
+
+        deleteLogs();
+    }
+
+    /** */
+    @Override protected void tearDown() throws Exception {
+        super.tearDown();
+
+        deleteLogs();
+    }
+
+    /** */
+    public void testMarkerFiltering() throws Exception {
+        // create log
+        Slf4jLogger log = new Slf4jLogger(LoggerFactory.getLogger(Slf4jLoggerMarkerTest.class));
+
+        // populate log with messages
+        log.error("IGNORE_ME", "Ignored error", null);
+        log.warning("IGNORE_ME", "Ignored warning", null);
+        log.info("IGNORE_ME", "Ignored info");
+        log.debug("IGNORE_ME", "Ignored debug");
+        log.trace("IGNORE_ME", "Ignored trace");
+
+        log.error("ACCEPT_ME", "Accepted error", null);
+        log.warning("ACCEPT_ME", "Accepted warning", null);
+        log.info("ACCEPT_ME", "Accepted info");
+        log.debug("ACCEPT_ME", "Accepted debug");
+        log.trace("ACCEPT_ME", "Accepted trace");
+
+        // check file with all messages
+        File allFile = U.resolveIgnitePath(LOG_ALL);
+        assertNotNull(allFile);
+        String all = U.readFileToString(allFile.getPath(), "UTF-8");
+
+        assertTrue(all.contains("[IGNORE_ME] Ignored error"));
+        assertTrue(all.contains("[IGNORE_ME] Ignored warning"));
+        assertTrue(all.contains("[IGNORE_ME] Ignored info"));
+        assertTrue(all.contains("[IGNORE_ME] Ignored debug"));
+        assertTrue(all.contains("[IGNORE_ME] Ignored trace"));
+
+        assertTrue(all.contains("[ACCEPT_ME] Accepted error"));
+        assertTrue(all.contains("[ACCEPT_ME] Accepted warning"));
+        assertTrue(all.contains("[ACCEPT_ME] Accepted info"));
+        assertTrue(all.contains("[ACCEPT_ME] Accepted debug"));
+        assertTrue(all.contains("[ACCEPT_ME] Accepted trace"));
+
+        // check file with one marker filtered out
+        File filteredFile = U.resolveIgnitePath(LOG_FILTERED);
+        assertNotNull(filteredFile);
+        String filtered = U.readFileToString(filteredFile.getPath(), "UTF-8");
+
+        assertFalse(filtered.contains("[IGNORE_ME] Ignored error"));
+        assertFalse(filtered.contains("[IGNORE_ME] Ignored warning"));
+        assertFalse(filtered.contains("[IGNORE_ME] Ignored info"));
+        assertFalse(filtered.contains("[IGNORE_ME] Ignored debug"));
+        assertFalse(filtered.contains("[IGNORE_ME] Ignored trace"));
+
+        assertTrue(filtered.contains("[ACCEPT_ME] Accepted error"));
+        assertTrue(filtered.contains("[ACCEPT_ME] Accepted warning"));
+        assertTrue(filtered.contains("[ACCEPT_ME] Accepted info"));
+        assertTrue(filtered.contains("[ACCEPT_ME] Accepted debug"));
+        assertTrue(filtered.contains("[ACCEPT_ME] Accepted trace"));
+    }
+
+    /** Delete existing logs, if any */
+    @SuppressWarnings("ResultOfMethodCallIgnored")
+    private void deleteLogs() {
+        new File(LOG_ALL).delete();
+        new File(LOG_FILTERED).delete();
+    }
+}
diff --git a/modules/slf4j/src/test/java/org/apache/ignite/testsuites/IgniteSlf4jTestSuite.java b/modules/slf4j/src/test/java/org/apache/ignite/testsuites/IgniteSlf4jTestSuite.java
new file mode 100644
index 0000000000000..8668f927ca370
--- /dev/null
+++ b/modules/slf4j/src/test/java/org/apache/ignite/testsuites/IgniteSlf4jTestSuite.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.logger.slf4j.Slf4jLoggerMarkerTest;
+
+/**
+ * Slf4j logging tests.
+ */
+public class IgniteSlf4jTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     */
+    public static TestSuite suite() {
+        TestSuite suite = new TestSuite("Slf4j Logging Test Suite");
+
+        suite.addTestSuite(Slf4jLoggerMarkerTest.class);
+
+        return suite;
+    }
+}
diff --git a/modules/slf4j/src/test/resources/log4j2-test.xml b/modules/slf4j/src/test/resources/log4j2-test.xml
new file mode 100644
index 0000000000000..d30736e3197ca
--- /dev/null
+++ b/modules/slf4j/src/test/resources/log4j2-test.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<Configuration>
+    <Appenders>
+        <File name="FILTERED" fileName="work/log/filtered.log" append="false">
+            <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
+            <MarkerFilter marker="IGNORE_ME" onMatch="DENY" onMismatch="NEUTRAL"/>
+        </File>
+
+        <File name="ALL" fileName="work/log/all.log"  append="false">
+            <PatternLayout pattern="[%d{ISO8601}][%-5p][%t][%c{1}]%notEmpty{[%markerSimpleName]} %m%n"/>
+        </File>
+    </Appenders>
+
+    <Loggers>
+        <Root level="TRACE">
+            <AppenderRef ref="FILTERED"/>
+            <AppenderRef ref="ALL"/>
+        </Root>
+    </Loggers>
+</Configuration>
diff --git a/parent/pom.xml b/parent/pom.xml
index 391415a957805..c34026efb7d79 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -88,6 +88,7 @@
         <jtidy.version>r938</jtidy.version>
         <kafka.version>0.10.0.1</kafka.version>
         <karaf.version>4.0.2</karaf.version>
+        <log4j.version>2.6</log4j.version>
         <lucene.bundle.version>3.5.0_1</lucene.bundle.version>
         <lucene.version>3.5.0</lucene.version>
         <oro.bundle.version>2.0.8_6</oro.bundle.version>

From 837ee8cf1563d70dbe29a30d3ec2a83b99e5fe9b Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Mon, 22 Jan 2018 13:05:20 +0300
Subject: [PATCH 477/516] Fix copyright.

---
 .../internal/util/IgniteDevOnlyLogTest.java   | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteDevOnlyLogTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteDevOnlyLogTest.java
index 87b44da7f2cfe..101cb3c55d798 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteDevOnlyLogTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteDevOnlyLogTest.java
@@ -1,10 +1,18 @@
-/* @java.file.header */
-
-/*  _________        _____ __________________        _____
- *  __  ____/___________(_)______  /__  ____/______ ____(_)_______
- *  _  / __  __  ___/__  / _  __  / _  / __  _  __ `/__  / __  __ \
- *  / /_/ /  _  /    _  /  / /_/ /  / /_/ /  / /_/ / _  /  _  / / /
- *  \____/   /_/     /_/   \_,__/   \____/   \__,_/  /_/   /_/ /_/
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package org.apache.ignite.internal.util;

From d1ef1dfc783621e1172618e4257f53ef6332ddda Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Mon, 22 Jan 2018 14:43:30 +0300
Subject: [PATCH 478/516] GG-13356: Fix GridMBeanDisableSelfTest.

(cherry picked from commit e2bd8a6)
---
 .../src/main/java/org/apache/ignite/internal/IgniteKernal.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 17d696acce307..c12bf36ce06c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1674,6 +1674,9 @@ private ObjectName registerExecutorMBean(ExecutorService exec, String name) thro
      * @throws IgniteCheckedException If registration failed.
      */
     private void registerStripedExecutorMBean(StripedExecutor stripedExecSvc) throws IgniteCheckedException {
+        if(U.IGNITE_MBEANS_DISABLED)
+            return;
+
         if (stripedExecSvc != null) {
             String name = "StripedExecutor";
 

From b8f49fd52ba6d273b4e4824573e87a37628731b0 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Wed, 17 Jan 2018 14:40:50 +0300
Subject: [PATCH 479/516] Updated copyrights 2017 -> 2018.

(cherry picked from commit b234640)
---
 NOTICE                                                          | 2 +-
 assembly/NOTICE_FABRIC                                          | 2 +-
 assembly/NOTICE_HADOOP                                          | 2 +-
 modules/core/src/main/java/META-INF/NOTICE                      | 2 +-
 .../java/org/apache/ignite/internal/IgniteVersionUtils.java     | 2 +-
 .../org/apache/ignite/startup/GridRandomCommandLineLoader.java  | 2 +-
 .../Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs       | 2 +-
 .../dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec     | 2 +-
 .../dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs      | 2 +-
 .../dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs  | 2 +-
 .../Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs   | 2 +-
 .../Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs | 2 +-
 .../dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs  | 2 +-
 .../dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec  | 2 +-
 .../dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec         | 2 +-
 .../dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs        | 2 +-
 .../Apache.Ignite.EntityFramework.nuspec                        | 2 +-
 .../Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs    | 2 +-
 .../dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec         | 2 +-
 .../dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs        | 2 +-
 .../dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec   | 2 +-
 .../dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs     | 2 +-
 .../dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec         | 2 +-
 .../dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs        | 2 +-
 .../platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs   | 2 +-
 .../examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs  | 2 +-
 .../Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs        | 2 +-
 modules/scalar-2.10/pom.xml                                     | 2 +-
 modules/scalar/pom.xml                                          | 2 +-
 parent/pom.xml                                                  | 2 +-
 30 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/NOTICE b/NOTICE
index 33e2479d6e89e..4c99a05109185 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2017 The Apache Software Foundation
+Copyright 2018 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/assembly/NOTICE_FABRIC b/assembly/NOTICE_FABRIC
index c5e6f023ad023..964fd359ad713 100644
--- a/assembly/NOTICE_FABRIC
+++ b/assembly/NOTICE_FABRIC
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2017 The Apache Software Foundation
+Copyright 2018 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/assembly/NOTICE_HADOOP b/assembly/NOTICE_HADOOP
index 33e2479d6e89e..4c99a05109185 100644
--- a/assembly/NOTICE_HADOOP
+++ b/assembly/NOTICE_HADOOP
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2017 The Apache Software Foundation
+Copyright 2018 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/modules/core/src/main/java/META-INF/NOTICE b/modules/core/src/main/java/META-INF/NOTICE
index 33e2479d6e89e..4c99a05109185 100644
--- a/modules/core/src/main/java/META-INF/NOTICE
+++ b/modules/core/src/main/java/META-INF/NOTICE
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2017 The Apache Software Foundation
+Copyright 2018 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java
index 7cb15bec5f8ee..8a459522e70b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java
@@ -47,7 +47,7 @@ public class IgniteVersionUtils {
     public static final String ACK_VER_STR;
 
     /** Copyright blurb. */
-    public static final String COPYRIGHT = "2017 Copyright(C) Apache Software Foundation";
+    public static final String COPYRIGHT = "2018 Copyright(C) Apache Software Foundation";
 
     /**
      * Static initializer.
diff --git a/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java b/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java
index 73a9760a91536..9284b8a327c2a 100644
--- a/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java
+++ b/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java
@@ -61,7 +61,7 @@ public final class GridRandomCommandLineLoader {
     private static final String IGNITE_PROG_NAME = "IGNITE_PROG_NAME";
 
     /** Copyright text. Ant processed. */
-    private static final String COPYRIGHT = "2017 Copyright(C) Apache Software Foundation.";
+    private static final String COPYRIGHT = "2018 Copyright(C) Apache Software Foundation.";
 
     /** Version. Ant processed. */
     private static final String VER = "x.x.x";
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
index 69815008a79b1..3566004023b17 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
@@ -27,7 +27,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec b/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec
index 7891614e5d26c..3f795eb4bbf94 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec
@@ -45,7 +45,7 @@ Session State Store Provider: stores session state data in a distributed in-memo
 More info: https://apacheignite-net.readme.io/
         </description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2017</copyright>
+        <copyright>Copyright 2018</copyright>
         <tags>OutputCacheProvider Apache Ignite In-Memory Distributed Computing SQL NoSQL Grid Map Reduce Cache</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
index 1073986202465..cc13749bcc7a6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
@@ -25,7 +25,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
index ce822816457d9..533f643f07b85 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
index d6b169916387d..78b340cf4a822 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
index e9f93bd4dc010..ee8a59dae2e6b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
index 28f6e72041909..43794249e129b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec
index 52dc1e03f1c1e..a6c999c7ac263 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec
@@ -42,7 +42,7 @@ XSD file describes the structure of IgniteConfigurationSection and enables Intel
 More info on Apache Ignite.NET: https://apacheignite-net.readme.io/
         </description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2017</copyright>
+        <copyright>Copyright 2018</copyright>
         <tags>Apache Ignite XSD Intellisense</tags>
     </metadata>
     <files>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
index 8f562f109ced6..364895515fbb2 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
@@ -45,7 +45,7 @@ Apache Ignite In-Memory Data Fabric is a high-performance, integrated and distri
 More info: https://apacheignite-net.readme.io/
         </description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2017</copyright>
+        <copyright>Copyright 2018</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL Grid Map Reduce Cache linqpad-samples</tags>
     </metadata>
     <files>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
index cb903aad8f841..a07d75a8a2ccc 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
@@ -25,7 +25,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Apache.Ignite.EntityFramework.nuspec b/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Apache.Ignite.EntityFramework.nuspec
index 477dcf875206d..84069b31ea4f4 100644
--- a/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Apache.Ignite.EntityFramework.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Apache.Ignite.EntityFramework.nuspec
@@ -47,7 +47,7 @@ More info: https://apacheignite-net.readme.io/
             Apache Ignite EntityFramework Integration
         </summary>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2017</copyright>
+        <copyright>Copyright 2018</copyright>
         <tags>EntityFramework Second-Level Apache Ignite In-Memory Distributed Computing SQL NoSQL Grid Map Reduce Cache</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs
index 4d25f53f8099b..b8d62bba28b6e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs
@@ -25,7 +25,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec b/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec
index c71d672c10c76..d956775d21eb8 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec
@@ -47,7 +47,7 @@ All Ignite SQL features are supported: distributed joins, groupings, aggregates,
 More info: https://apacheignite-net.readme.io/
         </description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2017</copyright>
+        <copyright>Copyright 2018</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL LINQ Grid Map Reduce Cache linqpad-samples</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
index 561674a13e6dc..4597e8eb01be5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
@@ -24,7 +24,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec
index a3f86c17cc88f..1193cff38c77d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec
@@ -40,7 +40,7 @@ Creating NuGet package:
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <description>log4net Logger for Apache Ignite</description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2017</copyright>
+        <copyright>Copyright 2018</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL LINQ Grid Map Reduce Cache log4net logger</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
index 09e57492b149a..477c6009a7728 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
@@ -24,7 +24,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec b/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec
index e3a6f42db4c4a..58f9ffbd53f5e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec
@@ -40,7 +40,7 @@ Creating NuGet package:
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <description>NLog Logger for Apache Ignite</description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2017</copyright>
+        <copyright>Copyright 2018</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL LINQ Grid Map Reduce Cache NLog logger</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
index 3f2aa714044aa..f5b984eea0526 100644
--- a/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
@@ -24,7 +24,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
index 3e0e305c48dc6..563b0b7d71647 100644
--- a/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
index f6323e06f39bc..ea67c333f309e 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
index 6d28b2bb9c7b2..0c43f1ac1c32e 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2017")]
+[assembly: AssemblyCopyright("Copyright 2018")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/scalar-2.10/pom.xml b/modules/scalar-2.10/pom.xml
index 8d3d86d9503b4..b3b75b3cc5b59 100644
--- a/modules/scalar-2.10/pom.xml
+++ b/modules/scalar-2.10/pom.xml
@@ -158,7 +158,7 @@
                                             <!--<td>-->
                                                 <!--<nobr>Ignite&#153; - Scalar DSL, ver. <strong>${project.version}</strong></nobr>-->
                                                 <!--<br>-->
-                                                <!--<a target=_blank href="https://ignite.apache.org"><nobr>2017 Copyright &#169; Apache Software Foundation</nobr></a>-->
+                                                <!--<a target=_blank href="https://ignite.apache.org"><nobr>2018 Copyright &#169; Apache Software Foundation</nobr></a>-->
                                             <!--</td>-->
                                         <!--</tr>-->
                                         <!--</table>-->
diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml
index 2bb114fb33c6c..f5f58e06cf61e 100644
--- a/modules/scalar/pom.xml
+++ b/modules/scalar/pom.xml
@@ -143,7 +143,7 @@
                                             <td>
                                                 <nobr>Ignite&#153; - Scalar DSL, ver. <strong>${project.version}</strong></nobr>
                                                 <br>
-                                                <a target=_blank href="https://ignite.apache.org"><nobr>2017 Copyright &#169; Apache Software Foundation</nobr></a>
+                                                <a target=_blank href="https://ignite.apache.org"><nobr>2018 Copyright &#169; Apache Software Foundation</nobr></a>
                                             </td>
                                         </tr>
                                         </table>
diff --git a/parent/pom.xml b/parent/pom.xml
index c34026efb7d79..6ff39894d1d49 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -466,7 +466,7 @@
                                                 <tbody style="padding: 0; margin: 0">
                                                     <tr style="padding: 0; margin: 0">
                                                         <td>
-                                                            <a target=_blank href="https://ignite.apache.org"><nobr>2017 Copyright &#169; Apache Software Foundation</nobr></a>
+                                                            <a target=_blank href="https://ignite.apache.org"><nobr>2018 Copyright &#169; Apache Software Foundation</nobr></a>
                                                         </td>
                                                     </tr>
                                                 </tbody>

From de7edfcf1ba75e071edfdec3ec3322c4c9eee1eb Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Wed, 24 Jan 2018 15:54:00 +0300
Subject: [PATCH 480/516] Revert "Updated copyrights 2017 -> 2018."

This reverts commit b8f49fd
---
 NOTICE                                                          | 2 +-
 assembly/NOTICE_FABRIC                                          | 2 +-
 assembly/NOTICE_HADOOP                                          | 2 +-
 modules/core/src/main/java/META-INF/NOTICE                      | 2 +-
 .../java/org/apache/ignite/internal/IgniteVersionUtils.java     | 2 +-
 .../org/apache/ignite/startup/GridRandomCommandLineLoader.java  | 2 +-
 .../Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs       | 2 +-
 .../dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec     | 2 +-
 .../dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs      | 2 +-
 .../dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs  | 2 +-
 .../Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs   | 2 +-
 .../Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs | 2 +-
 .../dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs  | 2 +-
 .../dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec  | 2 +-
 .../dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec         | 2 +-
 .../dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs        | 2 +-
 .../Apache.Ignite.EntityFramework.nuspec                        | 2 +-
 .../Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs    | 2 +-
 .../dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec         | 2 +-
 .../dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs        | 2 +-
 .../dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec   | 2 +-
 .../dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs     | 2 +-
 .../dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec         | 2 +-
 .../dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs        | 2 +-
 .../platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs   | 2 +-
 .../examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs  | 2 +-
 .../Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs        | 2 +-
 modules/scalar-2.10/pom.xml                                     | 2 +-
 modules/scalar/pom.xml                                          | 2 +-
 parent/pom.xml                                                  | 2 +-
 30 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/NOTICE b/NOTICE
index 4c99a05109185..33e2479d6e89e 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2018 The Apache Software Foundation
+Copyright 2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/assembly/NOTICE_FABRIC b/assembly/NOTICE_FABRIC
index 964fd359ad713..c5e6f023ad023 100644
--- a/assembly/NOTICE_FABRIC
+++ b/assembly/NOTICE_FABRIC
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2018 The Apache Software Foundation
+Copyright 2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/assembly/NOTICE_HADOOP b/assembly/NOTICE_HADOOP
index 4c99a05109185..33e2479d6e89e 100644
--- a/assembly/NOTICE_HADOOP
+++ b/assembly/NOTICE_HADOOP
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2018 The Apache Software Foundation
+Copyright 2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/modules/core/src/main/java/META-INF/NOTICE b/modules/core/src/main/java/META-INF/NOTICE
index 4c99a05109185..33e2479d6e89e 100644
--- a/modules/core/src/main/java/META-INF/NOTICE
+++ b/modules/core/src/main/java/META-INF/NOTICE
@@ -1,5 +1,5 @@
 Apache Ignite
-Copyright 2018 The Apache Software Foundation
+Copyright 2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java
index 8a459522e70b2..7cb15bec5f8ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteVersionUtils.java
@@ -47,7 +47,7 @@ public class IgniteVersionUtils {
     public static final String ACK_VER_STR;
 
     /** Copyright blurb. */
-    public static final String COPYRIGHT = "2018 Copyright(C) Apache Software Foundation";
+    public static final String COPYRIGHT = "2017 Copyright(C) Apache Software Foundation";
 
     /**
      * Static initializer.
diff --git a/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java b/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java
index 9284b8a327c2a..73a9760a91536 100644
--- a/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java
+++ b/modules/core/src/test/java/org/apache/ignite/startup/GridRandomCommandLineLoader.java
@@ -61,7 +61,7 @@ public final class GridRandomCommandLineLoader {
     private static final String IGNITE_PROG_NAME = "IGNITE_PROG_NAME";
 
     /** Copyright text. Ant processed. */
-    private static final String COPYRIGHT = "2018 Copyright(C) Apache Software Foundation.";
+    private static final String COPYRIGHT = "2017 Copyright(C) Apache Software Foundation.";
 
     /** Version. Ant processed. */
     private static final String VER = "x.x.x";
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
index 3566004023b17..69815008a79b1 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/Properties/AssemblyInfo.cs
@@ -27,7 +27,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec b/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec
index 3f795eb4bbf94..7891614e5d26c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet/Apache.Ignite.AspNet.nuspec
@@ -45,7 +45,7 @@ Session State Store Provider: stores session state data in a distributed in-memo
 More info: https://apacheignite-net.readme.io/
         </description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2018</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>OutputCacheProvider Apache Ignite In-Memory Distributed Computing SQL NoSQL Grid Map Reduce Cache</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
index cc13749bcc7a6..1073986202465 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet/Properties/AssemblyInfo.cs
@@ -25,7 +25,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
index 533f643f07b85..ce822816457d9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
index 78b340cf4a822..d6b169916387d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.NuGet/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
index ee8a59dae2e6b..e9f93bd4dc010 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
index 43794249e129b..28f6e72041909 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec
index a6c999c7ac263..52dc1e03f1c1e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.Schema.nuspec
@@ -42,7 +42,7 @@ XSD file describes the structure of IgniteConfigurationSection and enables Intel
 More info on Apache Ignite.NET: https://apacheignite-net.readme.io/
         </description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2018</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>Apache Ignite XSD Intellisense</tags>
     </metadata>
     <files>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
index 364895515fbb2..8f562f109ced6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec
@@ -45,7 +45,7 @@ Apache Ignite In-Memory Data Fabric is a high-performance, integrated and distri
 More info: https://apacheignite-net.readme.io/
         </description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2018</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL Grid Map Reduce Cache linqpad-samples</tags>
     </metadata>
     <files>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
index a07d75a8a2ccc..cb903aad8f841 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs
@@ -25,7 +25,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Apache.Ignite.EntityFramework.nuspec b/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Apache.Ignite.EntityFramework.nuspec
index 84069b31ea4f4..477dcf875206d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Apache.Ignite.EntityFramework.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Apache.Ignite.EntityFramework.nuspec
@@ -47,7 +47,7 @@ More info: https://apacheignite-net.readme.io/
             Apache Ignite EntityFramework Integration
         </summary>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2018</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>EntityFramework Second-Level Apache Ignite In-Memory Distributed Computing SQL NoSQL Grid Map Reduce Cache</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs
index b8d62bba28b6e..4d25f53f8099b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.EntityFramework/Properties/AssemblyInfo.cs
@@ -25,7 +25,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec b/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec
index d956775d21eb8..c71d672c10c76 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.nuspec
@@ -47,7 +47,7 @@ All Ignite SQL features are supported: distributed joins, groupings, aggregates,
 More info: https://apacheignite-net.readme.io/
         </description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2018</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL LINQ Grid Map Reduce Cache linqpad-samples</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
index 4597e8eb01be5..561674a13e6dc 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Properties/AssemblyInfo.cs
@@ -24,7 +24,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec
index 1193cff38c77d..a3f86c17cc88f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Apache.Ignite.Log4Net.nuspec
@@ -40,7 +40,7 @@ Creating NuGet package:
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <description>log4net Logger for Apache Ignite</description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2018</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL LINQ Grid Map Reduce Cache log4net logger</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
index 477c6009a7728..09e57492b149a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Log4Net/Properties/AssemblyInfo.cs
@@ -24,7 +24,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec b/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec
index 58f9ffbd53f5e..e3a6f42db4c4a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec
+++ b/modules/platforms/dotnet/Apache.Ignite.NLog/Apache.Ignite.NLog.nuspec
@@ -40,7 +40,7 @@ Creating NuGet package:
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <description>NLog Logger for Apache Ignite</description>
         <releaseNotes></releaseNotes>
-        <copyright>Copyright 2018</copyright>
+        <copyright>Copyright 2017</copyright>
         <tags>Apache Ignite In-Memory Distributed Computing SQL NoSQL LINQ Grid Map Reduce Cache NLog logger</tags>
         <dependencies>
             <dependency id="Apache.Ignite" version="[$version$]" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
index f5b984eea0526..3f2aa714044aa 100644
--- a/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.NLog/Properties/AssemblyInfo.cs
@@ -24,7 +24,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
index 563b0b7d71647..3e0e305c48dc6 100644
--- a/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
index ea67c333f309e..f6323e06f39bc 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs b/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
index 0c43f1ac1c32e..6d28b2bb9c7b2 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs
@@ -23,7 +23,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("Apache Software Foundation")]
 [assembly: AssemblyProduct("Apache Ignite.NET")]
-[assembly: AssemblyCopyright("Copyright 2018")]
+[assembly: AssemblyCopyright("Copyright 2017")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
diff --git a/modules/scalar-2.10/pom.xml b/modules/scalar-2.10/pom.xml
index b3b75b3cc5b59..8d3d86d9503b4 100644
--- a/modules/scalar-2.10/pom.xml
+++ b/modules/scalar-2.10/pom.xml
@@ -158,7 +158,7 @@
                                             <!--<td>-->
                                                 <!--<nobr>Ignite&#153; - Scalar DSL, ver. <strong>${project.version}</strong></nobr>-->
                                                 <!--<br>-->
-                                                <!--<a target=_blank href="https://ignite.apache.org"><nobr>2018 Copyright &#169; Apache Software Foundation</nobr></a>-->
+                                                <!--<a target=_blank href="https://ignite.apache.org"><nobr>2017 Copyright &#169; Apache Software Foundation</nobr></a>-->
                                             <!--</td>-->
                                         <!--</tr>-->
                                         <!--</table>-->
diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml
index f5f58e06cf61e..2bb114fb33c6c 100644
--- a/modules/scalar/pom.xml
+++ b/modules/scalar/pom.xml
@@ -143,7 +143,7 @@
                                             <td>
                                                 <nobr>Ignite&#153; - Scalar DSL, ver. <strong>${project.version}</strong></nobr>
                                                 <br>
-                                                <a target=_blank href="https://ignite.apache.org"><nobr>2018 Copyright &#169; Apache Software Foundation</nobr></a>
+                                                <a target=_blank href="https://ignite.apache.org"><nobr>2017 Copyright &#169; Apache Software Foundation</nobr></a>
                                             </td>
                                         </tr>
                                         </table>
diff --git a/parent/pom.xml b/parent/pom.xml
index 6ff39894d1d49..c34026efb7d79 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -466,7 +466,7 @@
                                                 <tbody style="padding: 0; margin: 0">
                                                     <tr style="padding: 0; margin: 0">
                                                         <td>
-                                                            <a target=_blank href="https://ignite.apache.org"><nobr>2018 Copyright &#169; Apache Software Foundation</nobr></a>
+                                                            <a target=_blank href="https://ignite.apache.org"><nobr>2017 Copyright &#169; Apache Software Foundation</nobr></a>
                                                         </td>
                                                     </tr>
                                                 </tbody>

From bc22a8ab430044f84b91d8d294a1cbd413d9af53 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 30 Jan 2018 12:33:51 +0300
Subject: [PATCH 481/516] GG-13127 - Mute JDBC tests

---
 ...acheJdbcPojoStoreMultitreadedSelfTest.java | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreMultitreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreMultitreadedSelfTest.java
index a662d4ef7a1a8..2dd57720229fd 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreMultitreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreMultitreadedSelfTest.java
@@ -17,6 +17,9 @@
 
 package org.apache.ignite.cache.store.jdbc;
 
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.h2.jdbcx.JdbcConnectionPool;
 
 /**
@@ -32,4 +35,37 @@ public class CacheJdbcPojoStoreMultitreadedSelfTest
 
         return store;
     }
+
+    /** {@inheritDoc} */
+    @Override public void testMultithreadedPut() throws Exception {
+        if (!supportedMarshaller())
+            fail("https://ggsystems.atlassian.net/browse/GG-13127");
+
+        super.testMultithreadedPut();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void testMultithreadedPutAll() throws Exception {
+        if (!supportedMarshaller())
+            fail("https://ggsystems.atlassian.net/browse/GG-13127");
+
+        super.testMultithreadedPutAll();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void testMultithreadedExplicitTx() throws Exception {
+        if (!supportedMarshaller())
+            fail("https://ggsystems.atlassian.net/browse/GG-13127");
+
+        super.testMultithreadedExplicitTx();
+    }
+
+    /**
+     * @return Supported marshaller.
+     */
+    private boolean supportedMarshaller() {
+        Marshaller marsh = grid().configuration().getMarshaller();
+
+        return marsh instanceof BinaryMarshaller || marsh instanceof OptimizedMarshaller;
+    }
 }
\ No newline at end of file

From b11c9bb774ce50d50e618ee3cb2bd489473d5f2d Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 30 Jan 2018 13:38:34 +0300
Subject: [PATCH 482/516] GG-13132 - Mute testOnlyOneTtlCleanupThreadExists for
 PortableMarshaller

---
 ...iteCacheOnlyOneTtlCleanupThreadExistsTest.java | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheOnlyOneTtlCleanupThreadExistsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheOnlyOneTtlCleanupThreadExistsTest.java
index 84f5144aef5b1..d31955ea2e2ba 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheOnlyOneTtlCleanupThreadExistsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheOnlyOneTtlCleanupThreadExistsTest.java
@@ -19,6 +19,9 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -36,6 +39,9 @@ public class IgniteCacheOnlyOneTtlCleanupThreadExistsTest extends GridCommonAbst
      * @throws Exception If failed.
      */
     public void testOnlyOneTtlCleanupThreadExists() throws Exception {
+        if (!supportedMarshaller())
+            fail("https://ggsystems.atlassian.net/browse/GG-13132");
+
         try (final Ignite g = startGrid(0)) {
             checkCleanupThreadExists(false);
 
@@ -99,4 +105,13 @@ private void checkCleanupThreadExists(boolean exists) throws Exception {
         else
             assertEquals("Ttl cleanup thread exists", cnt, 0);
     }
+
+    /**
+     * @return Supported marshaller.
+     */
+    private boolean supportedMarshaller() {
+        Marshaller marsh = grid().configuration().getMarshaller();
+
+        return marsh instanceof BinaryMarshaller || marsh instanceof OptimizedMarshaller;
+    }
 }

From 4afa1b2d2c99165671027584b619dc146cf6cb9d Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Tue, 27 Feb 2018 19:38:10 +0300
Subject: [PATCH 483/516] GG-13518 Fixed error when using TreeMap or TreeSet as
 cache key.

---
 .../ignite/internal/binary/BinaryTreeMap.java |   3 +
 .../ignite/internal/binary/BinaryTreeSet.java |   3 +
 .../internal/binary/BinaryTreeSelfTest.java   | 284 +++++++++++++++---
 3 files changed, 248 insertions(+), 42 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java
index 3dae8ce74fa27..d556936eb8e2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeMap.java
@@ -32,6 +32,9 @@
 
 /**
  * Binary {@link TreeMap} wrapper.
+ *
+ * Note: It is required to supply a BinaryTypeConfiguration for BinaryTreeMap with BinaryArrayIdentityResolver
+ *  for using this type in keys.
  */
 public class BinaryTreeMap implements Binarylizable, Serializable {
     /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeSet.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeSet.java
index 2b01528848efb..49da0ab4b0365 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTreeSet.java
@@ -30,6 +30,9 @@
 
 /**
  * Binary {@link TreeSet} wrapper.
+ *
+ * Note: It is required to supply a BinaryTypeConfiguration for BinaryTreeSet with BinaryArrayIdentityResolver
+ *  for using this type in keys.
  */
 public class BinaryTreeSet implements Binarylizable {
     /** Original set. */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
index d57b34de7eab8..ee3bc2bc671c6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
@@ -17,10 +17,15 @@
 
 package org.apache.ignite.internal.binary;
 
+import java.util.Arrays;
+import javax.cache.Cache.Entry;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
+import org.apache.ignite.binary.BinaryArrayIdentityResolver;
 import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryTypeConfiguration;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.BinaryConfiguration;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.util.typedef.G;
@@ -71,8 +76,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      *
      * @throws Exception If failed.
      */
-    public void testTreeMapRegularNoComparator() throws Exception {
-        checkTreeMap(false, false);
+    public void testTreeMapAsValueRegularNoComparator() throws Exception {
+        checkTreeMapAsValue(false, false);
     }
 
     /**
@@ -80,8 +85,8 @@ public void testTreeMapRegularNoComparator() throws Exception {
      *
      * @throws Exception If failed.
      */
-    public void testTreeMapRegularComparator() throws Exception {
-        checkTreeMap(false, true);
+    public void testTreeMapAsValueRegularComparator() throws Exception {
+        checkTreeMapAsValue(false, true);
     }
 
     /**
@@ -89,8 +94,8 @@ public void testTreeMapRegularComparator() throws Exception {
      *
      * @throws Exception If failed.
      */
-    public void testTreeMapBinaryNoComparator() throws Exception {
-        checkTreeMap(true, false);
+    public void testTreeMapAsValueBinaryNoComparator() throws Exception {
+        checkTreeMapAsValue(true, false);
     }
 
     /**
@@ -98,8 +103,26 @@ public void testTreeMapBinaryNoComparator() throws Exception {
      *
      * @throws Exception If failed.
      */
-    public void testTreeMapBinaryComparator() throws Exception {
-        checkTreeMap(true, true);
+    public void testTreeMapAsValueBinaryComparator() throws Exception {
+        checkTreeMapAsValue(true, true);
+    }
+
+    /**
+     * Test {@code TreeMap} data structure when used as key.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTreeMapAsKeyNoComparator() throws Exception {
+        checkTreeMapAsKey(false);
+    }
+
+    /**
+     * Test {@code TreeMap} data structure with comparator when used as key.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTreeMapAsKeyComparator() throws Exception {
+        checkTreeMapAsKey(true);
     }
 
     /**
@@ -110,22 +133,11 @@ public void testTreeMapBinaryComparator() throws Exception {
      * @throws Exception If failed.
      */
     @SuppressWarnings("unchecked")
-    private void checkTreeMap(boolean useBinary, boolean useComparator) throws Exception {
+    private void checkTreeMapAsValue(boolean useBinary, boolean useComparator) throws Exception {
         // Populate map.
         TreeMap<TestKey, Integer> map;
 
-        if (useComparator) {
-            map = new TreeMap<>(new TestKeyComparator());
-
-            for (int i = 0; i < SIZE; i++)
-                map.put(key(false, i), i);
-        }
-        else {
-            map = new TreeMap<>();
-
-            for (int i = 0; i < SIZE; i++)
-                map.put(key(true, i), i);
-        }
+        map = testMap(useComparator);
 
         // Put and get value from cache.
         cache().put(KEY, map);
@@ -147,6 +159,71 @@ private void checkTreeMap(boolean useBinary, boolean useComparator) throws Excep
             assertNull(resMap.comparator());
 
         assertEquals(map, resMap);
+
+        cache().clear();
+    }
+
+    /**
+     * Check {@code TreeMap} data structure when used as key.
+     *
+     * @param useComparator Whether comparator should be used.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    private void checkTreeMapAsKey(boolean useComparator) throws Exception {
+        // Populate map.
+        TreeMap<TestKey, Integer> map;
+
+        map = testMap(useComparator);
+
+        // Put and get value from cache.
+        cache().put(map, KEY);
+
+        TreeMap<TestKey, Integer> resMap = (TreeMap<TestKey, Integer>)((Entry)cache().iterator().next()).getKey();
+
+        // Ensure content is correct.
+        if (useComparator)
+            assert resMap.comparator() instanceof TestKeyComparator;
+        else
+            assertNull(resMap.comparator());
+
+        assertEquals(map, resMap);
+
+        // Ensure value is correct.
+        Integer resSameMap = (Integer)cache().get(map);
+
+        assertEquals((Object)KEY, resSameMap);
+
+        Integer resIdenticalMap = (Integer)cache().get(testMap(useComparator));
+
+        assertEquals((Object)KEY, resIdenticalMap);
+
+        // Ensure wrong comparator is not accepted.
+        Integer resDifferentComparator = (Integer)cache().get(testMap(!useComparator));
+
+        assertEquals(null, resDifferentComparator);
+
+        cache().clear();
+    }
+
+    /** */
+    private TreeMap<TestKey, Integer> testMap(boolean useComparator) {
+        TreeMap<TestKey, Integer> map;
+
+        if (useComparator) {
+            map = new TreeMap<>(new TestKeyComparator());
+
+            for (int i = 0; i < SIZE; i++)
+                map.put(key(false, i), i);
+        }
+        else {
+            map = new TreeMap<>();
+
+            for (int i = 0; i < SIZE; i++)
+                map.put(key(true, i), i);
+        }
+
+        return map;
     }
 
     /**
@@ -154,8 +231,8 @@ private void checkTreeMap(boolean useBinary, boolean useComparator) throws Excep
      *
      * @throws Exception If failed.
      */
-    public void testTreeSetRegularNoComparator() throws Exception {
-        checkTreeSet(false, false);
+    public void testTreeSetAsValueRegularNoComparator() throws Exception {
+        checkTreeSetAsValue(false, false);
     }
 
     /**
@@ -163,8 +240,8 @@ public void testTreeSetRegularNoComparator() throws Exception {
      *
      * @throws Exception If failed.
      */
-    public void testTreeSetRegularComparator() throws Exception {
-        checkTreeSet(false, true);
+    public void testTreeSetAsValueRegularComparator() throws Exception {
+        checkTreeSetAsValue(false, true);
     }
 
     /**
@@ -172,8 +249,8 @@ public void testTreeSetRegularComparator() throws Exception {
      *
      * @throws Exception If failed.
      */
-    public void testTreeSetBinaryNoComparator() throws Exception {
-        checkTreeSet(true, false);
+    public void testTreeSetAsValueBinaryNoComparator() throws Exception {
+        checkTreeSetAsValue(true, false);
     }
 
     /**
@@ -181,8 +258,26 @@ public void testTreeSetBinaryNoComparator() throws Exception {
      *
      * @throws Exception If failed.
      */
-    public void testTreeSetBinaryComparator() throws Exception {
-        checkTreeSet(true, true);
+    public void testTreeSetAsValueBinaryComparator() throws Exception {
+        checkTreeSetAsValue(true, true);
+    }
+
+    /**
+     * Test {@code TreeSet} data structure.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTreeSetAsKeyNoComparator() throws Exception {
+        checkTreeSetAsKey(false);
+    }
+
+    /**
+     * Test {@code TreeSet} data structure with comparator.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTreeSetAsKeyComparator() throws Exception {
+        checkTreeSetAsKey(true);
     }
 
     /**
@@ -193,22 +288,11 @@ public void testTreeSetBinaryComparator() throws Exception {
      * @throws Exception If failed.
      */
     @SuppressWarnings("unchecked")
-    private void checkTreeSet(boolean useBinary, boolean useComparator) throws Exception {
+    private void checkTreeSetAsValue(boolean useBinary, boolean useComparator) throws Exception {
         // Populate set.
         TreeSet<TestKey> set;
 
-        if (useComparator) {
-            set = new TreeSet<>(new TestKeyComparator());
-
-            for (int i = 0; i < SIZE; i++)
-                set.add(key(false, i));
-        }
-        else {
-            set = new TreeSet<>();
-
-            for (int i = 0; i < SIZE; i++)
-                set.add(key(true, i));
-        }
+        set = testSet(useComparator);
 
         // Put and get value from cache.
         cache().put(KEY, set);
@@ -230,6 +314,74 @@ private void checkTreeSet(boolean useBinary, boolean useComparator) throws Excep
             assertNull(resSet.comparator());
 
         assertEquals(set, resSet);
+
+        cache().clear();
+    }
+
+
+    /**
+     * Check {@code TreeSet} data structure when used as key.
+     *
+     * @param useComparator Whether comparator should be used.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    private void checkTreeSetAsKey(boolean useComparator) throws Exception {
+        // Populate set.
+        TreeSet<TestKey> set;
+
+        set = testSet(useComparator);
+
+        // Put and get value from cache.
+        cache().put(set, KEY);
+
+        TreeSet<TestKey> resSet = (TreeSet<TestKey>)((Entry)cache().iterator().next()).getKey();
+
+        // Ensure content is correct.
+        if (useComparator)
+            assert resSet.comparator() instanceof TestKeyComparator;
+        else
+            assertNull(resSet.comparator());
+
+        assertEquals(set, resSet);
+
+        // Ensure value is correct.
+        Integer resSameMap = (Integer)cache().get(set);
+
+        assertEquals((Object)KEY, resSameMap);
+
+        Integer resIdenticalMap = (Integer)cache().get(testSet(useComparator));
+
+        assertEquals((Object)KEY, resIdenticalMap);
+
+        // Ensure wrong comparator is not accepted.
+        Integer resDifferentComparator = (Integer)cache().get(testSet(!useComparator));
+
+        assertEquals(null, resDifferentComparator);
+
+        assertEquals(set, resSet);
+
+        cache().clear();
+    }
+
+    /** */
+    private TreeSet<TestKey> testSet(boolean useComparator) {
+        TreeSet<TestKey> set;
+
+        if (useComparator) {
+            set = new TreeSet<>(new TestKeyComparator());
+
+            for (int i = 0; i < SIZE; i++)
+                set.add(key(false, i));
+        }
+        else {
+            set = new TreeSet<>();
+
+            for (int i = 0; i < SIZE; i++)
+                set.add(key(true, i));
+        }
+
+        return set;
     }
 
     /**
@@ -267,6 +419,22 @@ private static IgniteConfiguration configuration(String name, boolean client) {
         cfg.setLocalHost("127.0.0.1");
         cfg.setPeerClassLoadingEnabled(false);
 
+        BinaryTypeConfiguration mapTypeCfg = new BinaryTypeConfiguration();
+
+        mapTypeCfg.setTypeName("org.apache.ignite.internal.binary.BinaryTreeMap");
+        mapTypeCfg.setIdentityResolver(new BinaryArrayIdentityResolver());
+
+        BinaryTypeConfiguration setTypeCfg = new BinaryTypeConfiguration();
+
+        setTypeCfg.setTypeName("org.apache.ignite.internal.binary.BinaryTreeSet");
+        setTypeCfg.setIdentityResolver(new BinaryArrayIdentityResolver());
+
+        BinaryConfiguration binaryCfg = new BinaryConfiguration();
+
+        binaryCfg.setTypeConfigurations(Arrays.asList(mapTypeCfg, setTypeCfg));
+
+        cfg.setBinaryConfiguration(binaryCfg);
+
         TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
 
         TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
@@ -308,6 +476,23 @@ public TestKey(int id) {
         public int id() {
             return id;
         }
+
+        /** */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            TestKey key = (TestKey)o;
+
+            return id == key.id;
+        }
+
+        /** */
+        @Override public int hashCode() {
+            return id;
+        }
     }
 
     /**
@@ -337,5 +522,20 @@ private static class TestKeyComparator implements Comparator<TestKey> {
         @Override public int compare(TestKey o1, TestKey o2) {
             return o1.id() - o2.id();
         }
+
+        /** */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            return true;
+        }
+
+        /** */
+        @Override public int hashCode() {
+            return 13;
+        }
     }
 }

From 7146ce5698a157598c03b4e819ba6434932cda04 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Tue, 30 Jan 2018 13:38:34 +0300
Subject: [PATCH 484/516] IGNITE-5444, IGNITE-7718 Fixed serialization elements
 of singleton collections (cherry picked from commit 7da6ab97)

---
 .../apache/ignite/IgniteSystemProperties.java |  7 ++
 .../ignite/internal/binary/BinaryContext.java |  6 ++
 .../ignite/internal/binary/BinaryUtils.java   | 89 +++++++++++++++++--
 .../internal/binary/GridBinaryMarshaller.java |  9 ++
 .../processors/cache/CacheObjectContext.java  |  5 +-
 .../CacheObjectBinaryProcessorImpl.java       |  4 +-
 .../platform/utils/PlatformUtils.java         |  4 +-
 .../ignite/internal/util/IgniteUtils.java     | 37 ++++++++
 .../internal/util/MutableSingletonList.java   | 71 +++++++++++++++
 .../internal/util/MutableSingletonMap.java    | 64 +++++++++++++
 .../internal/util/MutableSingletonSet.java    | 86 ++++++++++++++++++
 ...BinaryObjectBuilderAdditionalSelfTest.java | 35 ++++++++
 ...ridCacheBinaryObjectsAbstractSelfTest.java | 82 +++++++++++++++++
 13 files changed, 487 insertions(+), 12 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonList.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonMap.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonSet.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 9d830dd908606..866c76d358afb 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -19,6 +19,7 @@
 
 import java.io.Serializable;
 import java.lang.management.RuntimeMXBean;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Properties;
@@ -604,6 +605,12 @@ public final class IgniteSystemProperties {
     /** Ignite marshaller cache reread pause. */
     public static final String IGNITE_MARSHALLER_CACHE_REREAD_PAUSE = "IGNITE_MARSHALLER_CACHE_REREAD_PAUSE";
 
+    /**
+     * Property allowing to serialize a singleton collections obtained by {@link Collections#singletonList(Object)},
+     * {@link Collections#singleton(Object)} and {@link Collections#singletonMap(Object, Object)}.
+     */
+    public static final String IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION = "IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION";
+
     /**
      * Class name of the closure {@link IgniteClosure}<String, Throwable> that
      * will be invoked once per topology and validates cache. If it returns or throws exception,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index bbf15bbcc4ecb..b1d0fcaceda8a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -277,6 +277,12 @@ public BinaryContext(BinaryMetadataHandler metaHnd, IgniteConfiguration igniteCf
         mapTypes.put(HashMap.class, GridBinaryMarshaller.HASH_MAP);
         mapTypes.put(LinkedHashMap.class, GridBinaryMarshaller.LINKED_HASH_MAP);
 
+        if (BinaryUtils.isSingletonCollectionSerializationEnabled()) {
+            colTypes.put(BinaryUtils.SINGLETON_LIST_CLS, GridBinaryMarshaller.SINGLETON_LIST);
+            colTypes.put(BinaryUtils.SINGLETON_SET_CLS, GridBinaryMarshaller.SINGLETON_SET);
+            mapTypes.put(BinaryUtils.SINGLETON_MAP_CLS, GridBinaryMarshaller.SINGLETON_MAP);
+        }
+
         // IDs range from [0..200] is used by Java SDK API and GridGain legacy API
 
         registerPredefinedType(Byte.class, GridBinaryMarshaller.BYTE);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 6831ef9bb0d66..957faa77f2b3b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -31,6 +31,7 @@
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -58,6 +59,9 @@
 import org.apache.ignite.binary.Binarylizable;
 import org.apache.ignite.internal.binary.builder.BinaryLazyValue;
 import org.apache.ignite.internal.binary.streams.BinaryInputStream;
+import org.apache.ignite.internal.util.MutableSingletonList;
+import org.apache.ignite.internal.util.MutableSingletonMap;
+import org.apache.ignite.internal.util.MutableSingletonSet;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -91,6 +95,15 @@ public class BinaryUtils {
     /** Binary classes. */
     private static final Collection<Class<?>> BINARY_CLS = new HashSet<>();
 
+    /** Class for SingletonList obtained at runtime. */
+    public static final Class<? extends Collection> SINGLETON_LIST_CLS = Collections.singletonList(null).getClass();
+
+    /** Class for SingletonMap obtained at runtime. */
+    public static final Class<? extends Map> SINGLETON_MAP_CLS = Collections.singletonMap(null, null).getClass();
+
+    /** Class for SingletonSet obtained at runtime. */
+    public static final Class<? extends Collection> SINGLETON_SET_CLS = Collections.singleton(null).getClass();
+
     /** Flag: user type. */
     public static final short FLAG_USR_TYP = 0x0001;
 
@@ -620,7 +633,8 @@ public static boolean knownMap(Object map) {
             cls == LinkedHashMap.class ||
             (!wrapTrees() && cls == TreeMap.class) ||
             cls == ConcurrentHashMap8.class ||
-            cls == ConcurrentHashMap.class;
+            cls == ConcurrentHashMap.class ||
+            (isSingletonCollectionSerializationEnabled() && cls == SINGLETON_MAP_CLS);
     }
 
     /**
@@ -644,6 +658,12 @@ else if (cls == ConcurrentHashMap8.class)
         else if (cls == ConcurrentHashMap.class)
             return new ConcurrentHashMap<>(U.capacity(((Map)map).size()));
 
+        if (isSingletonCollectionSerializationEnabled()) {
+            if(cls == SINGLETON_MAP_CLS){
+                return new MutableSingletonMap<>();
+            }
+        }
+
         return null;
     }
 
@@ -679,7 +699,8 @@ public static boolean knownCollection(Object col) {
             (!wrapTrees() && cls == TreeSet.class) ||
             cls == ConcurrentSkipListSet.class ||
             cls == ArrayList.class ||
-            cls == LinkedList.class;
+            cls == LinkedList.class ||
+            BinaryUtils.isSingletonCollection(cls);
     }
 
     /**
@@ -721,6 +742,15 @@ else if (cls == ArrayList.class)
         else if (cls == LinkedList.class)
             return new LinkedList<>();
 
+        if (BinaryUtils.isSingletonCollectionSerializationEnabled()) {
+            if (cls == SINGLETON_LIST_CLS) {
+                return new MutableSingletonList<>();
+            }
+            else if (cls == SINGLETON_SET_CLS) {
+                return new MutableSingletonSet<>();
+            }
+        }
+
         return null;
     }
 
@@ -1100,7 +1130,7 @@ else if (Proxy.class.isAssignableFrom(cls))
      * @return {@code True} if this is a special collection class.
      */
     public static boolean isSpecialCollection(Class cls) {
-        return ArrayList.class.equals(cls) || LinkedList.class.equals(cls) ||
+        return ArrayList.class.equals(cls) || LinkedList.class.equals(cls) || BinaryUtils.isSingletonCollection(cls) ||
             HashSet.class.equals(cls) || LinkedHashSet.class.equals(cls);
     }
 
@@ -1111,7 +1141,7 @@ public static boolean isSpecialCollection(Class cls) {
      * @return {@code True} if this is a special map class.
      */
     public static boolean isSpecialMap(Class cls) {
-        return HashMap.class.equals(cls) || LinkedHashMap.class.equals(cls);
+        return HashMap.class.equals(cls) || LinkedHashMap.class.equals(cls) || BinaryUtils.isSingletonMap(cls);
     }
 
     /**
@@ -1889,6 +1919,16 @@ public static Collection<?> doReadCollection(BinaryInputStream in, BinaryContext
 
                     break;
 
+                case GridBinaryMarshaller.SINGLETON_LIST:
+                    col = new MutableSingletonList<>();
+
+                    break;
+
+                case GridBinaryMarshaller.SINGLETON_SET:
+                    col = new MutableSingletonSet<>();
+
+                    break;
+
                 case GridBinaryMarshaller.HASH_SET:
                     col = U.newHashSet(size);
 
@@ -1919,7 +1959,7 @@ public static Collection<?> doReadCollection(BinaryInputStream in, BinaryContext
         for (int i = 0; i < size; i++)
             col.add(deserializeOrUnmarshal(in, ctx, ldr, handles, deserialize));
 
-        return col;
+        return U.unwrapSingletonCollection(col);
     }
 
     /**
@@ -1961,6 +2001,11 @@ public static Collection<?> doReadCollection(BinaryInputStream in, BinaryContext
 
                     break;
 
+                case GridBinaryMarshaller.SINGLETON_MAP:
+                    map = new MutableSingletonMap<>();
+
+                    break;
+
                 default:
                     throw new BinaryObjectException("Invalid map type: " + mapType);
             }
@@ -1975,7 +2020,7 @@ public static Collection<?> doReadCollection(BinaryInputStream in, BinaryContext
             map.put(key, val);
         }
 
-        return map;
+        return U.unwrapSingletonMap(map);
     }
 
     /**
@@ -2086,6 +2131,38 @@ public static void writeIgniteUuid(BinaryRawWriter out, @Nullable IgniteUuid val
             return null;
     }
 
+    /**
+     * Determines whether singleton collection serialization enabled.
+     *
+     * @return {@code true} if custom Java serialization logic exists, {@code false} otherwise.
+     * @see IgniteSystemProperties#IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION
+     */
+    public static boolean isSingletonCollectionSerializationEnabled() {
+        return Boolean.getBoolean(IgniteSystemProperties.IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION);
+    }
+
+    /**
+     * Determines whether target class could be serialized as singleton collection.
+     *
+     * @param cls target class.
+     * @return {@code True} if target class could be serialized as singleton collection.
+     */
+    public static boolean isSingletonCollection(Class cls) {
+        return cls != null && isSingletonCollectionSerializationEnabled() &&
+            (SINGLETON_LIST_CLS.equals(cls) || SINGLETON_SET_CLS.equals(cls));
+    }
+
+
+    /**
+     * Determines whether target map could be serialized as singleton map.
+     *
+     * @param cls target class.
+     * @return {@code True} if target class could be serialized as singleton map.
+     */
+    public static boolean isSingletonMap(Class cls) {
+        return cls != null && isSingletonCollectionSerializationEnabled() && SINGLETON_MAP_CLS.equals(cls);
+    }
+
     /**
      * Reconstructs string from UTF-8 bytes.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java
index 624fa33fd0f5d..503fa0c29adab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java
@@ -174,12 +174,21 @@ public class GridBinaryMarshaller {
     /** */
     public static final byte LINKED_HASH_SET = 4;
 
+    /** */
+    public static final byte SINGLETON_LIST = 5;
+
+    /** */
+    public static final byte SINGLETON_SET = 6;
+
     /** */
     public static final byte HASH_MAP = 1;
 
     /** */
     public static final byte LINKED_HASH_MAP = 2;
 
+    /** */
+    public static final byte SINGLETON_MAP = 3;
+
     /** */
     public static final byte PLATFORM_JAVA_OBJECT_FACTORY_PROXY = 99;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java
index c4203ef2421c2..d70300d7bc37a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.binary.BinaryUtils;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
  *
@@ -199,7 +200,7 @@ private Collection<Object> unwrapKnownCollection(Collection<Object> col, boolean
         for (Object obj : col)
             col0.add(unwrapBinary(obj, keepBinary, cpy));
 
-        return col0;
+        return U.unwrapSingletonCollection(col0);
     }
 
     /**
@@ -238,7 +239,7 @@ private Map<Object, Object> unwrapBinariesIfNeeded(Map<Object, Object> map, bool
         for (Map.Entry<Object, Object> e : map.entrySet())
             map0.put(unwrapBinary(e.getKey(), keepBinary, cpy), unwrapBinary(e.getValue(), keepBinary, cpy));
 
-        return map0;
+        return U.unwrapSingletonMap(map0);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 62fe328882a80..73ecb2abc91fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -508,7 +508,7 @@ public Object unmarshal(long ptr, boolean forceHeap) throws BinaryObjectExceptio
                 for (Object item : col)
                     pCol.add(marshalToBinary(item));
 
-                return pCol;
+                return U.unwrapSingletonCollection(pCol);
             }
         }
 
@@ -521,7 +521,7 @@ public Object unmarshal(long ptr, boolean forceHeap) throws BinaryObjectExceptio
                 for (Map.Entry<?, ?> e : map.entrySet())
                     pMap.put(marshalToBinary(e.getKey()), marshalToBinary(e.getValue()));
 
-                return pMap;
+                return U.unwrapSingletonMap(pMap);
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
index 0f1cd2cb60268..955bbca5eef99 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
@@ -951,7 +951,7 @@ private static Collection<Object> unwrapKnownCollection(Collection<Object> col)
         for (Object obj : col)
             col0.add(unwrapBinary(obj));
 
-        return col0;
+        return U.unwrapSingletonCollection(col0);
     }
 
     /**
@@ -981,7 +981,7 @@ private static Map<Object, Object> unwrapBinariesIfNeeded(Map<Object, Object> ma
         for (Map.Entry<Object, Object> e : map.entrySet())
             map0.put(unwrapBinary(e.getKey()), unwrapBinary(e.getValue()));
 
-        return map0;
+        return U.unwrapSingletonMap(map0);
     }
     /**
      * Create Java object.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 73055ae8bdab4..7d055aa5317b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -9415,6 +9415,43 @@ public static <K, V> Map<K, V> limitedMap(int limit) {
         return new HashMap<>(capacity(limit), 0.75f);
     }
 
+    /**
+     * Returns an immutable map if argument is singleton mutable map,
+     * otherwise returns argument.
+     *
+     * @param map map
+     * @param <K> type of map keys
+     * @param <V> type of map values
+     * @return argument or singleton map
+     */
+    public static <K,V> Map<K,V> unwrapSingletonMap(Map<K,V> map) {
+        if(map instanceof MutableSingletonMap){
+            return ((MutableSingletonMap<K,V>)map).singletonMap();
+        }
+
+        return map;
+    }
+
+    /**
+     * Returns an immutable collection if argument is singleton mutable collection,
+     * otherwise returns argument.
+     *
+     * @param col collection
+     * @param <T> type of collection elements
+     * @return argument or immutable singleton collection
+     */
+    public static <T> Collection<T> unwrapSingletonCollection(Collection<T> col) {
+        if(col instanceof MutableSingletonList){
+            return ((MutableSingletonList<T>)col).singletonList();
+        }
+
+        if(col instanceof MutableSingletonSet){
+            return ((MutableSingletonSet<T>)col).singletonSet();
+        }
+
+        return col;
+    }
+
     /**
      * Returns comparator that sorts remote node addresses. If remote node resides on the same host, then put
      * loopback addresses first, last otherwise.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonList.java
new file mode 100644
index 0000000000000..c15974143deb9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonList.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import java.util.AbstractList;
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * List that contains only one element. By default the list contains null element.
+ */
+public class MutableSingletonList<E> extends AbstractList<E> {
+
+    /** The only element of collection. */
+    private E element;
+
+    /** {@inheritDoc} */
+    @Override public E get(int index) {
+        if (index != 0)
+            throw new IndexOutOfBoundsException("Index: " + index + ", Size: " + size());
+
+        return element;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean add(E e) {
+        add(0, e);
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void add(int index, E element) {
+        if (index != 0)
+            throw new IllegalStateException("Element already added to singleton list");
+        else
+            this.element = element;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size() {
+        return 1;
+    }
+
+    /**
+     * Creates filled singleton list.
+     *
+     * @return singleton list
+     */
+    @SuppressWarnings("unchecked")
+    public Collection<E> singletonList() {
+        if (element == null) {
+            throw new IllegalStateException("Singleton list is empty");
+        }
+        return Collections.singletonList(element);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonMap.java b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonMap.java
new file mode 100644
index 0000000000000..52234e6dd2e77
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonMap.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import java.util.AbstractMap;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Map that contains only one key/value. By default the map contains null/null entry.
+ */
+public class MutableSingletonMap<K, V> extends AbstractMap<K, V> {
+
+    /** The key of single entry. */
+    protected K key;
+
+    /** The value of single entry. */
+    protected V value;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override public Set<Entry<K, V>> entrySet() {
+        return (Set)Collections.singleton(new SimpleEntry<>(key, value));
+    }
+
+    /** {@inheritDoc} */
+    @Override public V put(K key, V value) {
+        this.key = key;
+        this.value = value;
+
+        return value;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear() {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Creates filled singleton map.
+     *
+     * @return singleton map
+     */
+    @SuppressWarnings("unchecked")
+    public Map<K, V> singletonMap() {
+        return Collections.singletonMap(key, value);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonSet.java b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonSet.java
new file mode 100644
index 0000000000000..18b698b05fd42
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonSet.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import java.util.AbstractSet;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+/**
+ * Set that contains only one element. By default the set contains null element.
+ */
+public class MutableSingletonSet<E> extends AbstractSet<E> {
+
+    /** The only element of set. */
+    private E element;
+
+    /** {@inheritDoc} */
+    @Override public void clear() {
+        element = null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean add(E e) {
+        element = e;
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterator<E> iterator() {
+        return new Iterator<E>() {
+
+            private boolean hasNext = true;
+
+            @Override public boolean hasNext() {
+                return hasNext;
+            }
+
+            @Override public void remove() {
+                element = null;
+            }
+
+            @Override public E next() {
+                if (hasNext) {
+                    hasNext = false;
+
+                    return element;
+                }
+
+                throw new NoSuchElementException();
+            }
+        };
+    }
+
+    /** {@inheritDoc} */
+    @Override public int size() {
+        return 1;
+    }
+
+    /**
+     * Creates filled singleton set.
+     *
+     * @return singleton set
+     */
+    @SuppressWarnings("unchecked")
+    public Set<E> singletonSet() {
+        return Collections.singleton(element);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderAdditionalSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderAdditionalSelfTest.java
index 507aa6bf239ea..64de14a6207c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderAdditionalSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderAdditionalSelfTest.java
@@ -49,6 +49,7 @@
 import org.apache.ignite.IgniteBinary;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryObjectBuilder;
 import org.apache.ignite.binary.BinaryObjectException;
@@ -1513,6 +1514,7 @@ public void testCollectionsSerialization() {
 
         assert MAP.equals(binaryObj.type().fieldTypeName("singletonMap"));
 
+
         assert OBJ.equals(binaryObj.type().fieldTypeName("asList"));
         assert OBJ.equals(binaryObj.type().fieldTypeName("asSet"));
         assert OBJ.equals(binaryObj.type().fieldTypeName("asMap"));
@@ -1521,6 +1523,39 @@ public void testCollectionsSerialization() {
         assert OBJ.equals(binaryObj.type().fieldTypeName("asMapHint"));
     }
 
+    /**
+     * Check that correct type is stored in binary object.
+     */
+    public void testSingletonCollectionsSerialization() {
+        System.setProperty(IgniteSystemProperties.IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION, "true");
+        try {
+            final BinaryObjectBuilder root = newWrapper("SingletonColBinary");
+
+            final Set<Integer> hashSet = new HashSet<>();
+            hashSet.add(Integer.MAX_VALUE);
+
+            final Map<String, String> hashMap = new HashMap<>();
+            hashMap.put("key", "val");
+
+            // objects
+            root.setField("asList", Collections.singletonList(Integer.MAX_VALUE));
+            root.setField("asSet", Collections.singleton(Integer.MAX_VALUE));
+            root.setField("asMap", Collections.singletonMap("key", "val"));
+
+            BinaryObject binaryObj = root.build();
+
+            final String COL = "Collection";
+            final String MAP = "Map";
+            final String OBJ = "Object";
+
+            assert COL.equals(binaryObj.type().fieldTypeName("asList"));
+            assert COL.equals(binaryObj.type().fieldTypeName("asSet"));
+            assert MAP.equals(binaryObj.type().fieldTypeName("asMap"));
+        }finally {
+            System.clearProperty(IgniteSystemProperties.IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION);
+        }
+    }
+
     /**
      * Checks that externalizable value is correctly serialized/deserialized.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
index c9f50d164aa5b..dbc256eb3f797 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
@@ -21,6 +21,7 @@
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -37,6 +38,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteBinary;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryArrayIdentityResolver;
 import org.apache.ignite.binary.BinaryNameMapper;
 import org.apache.ignite.binary.BinaryObject;
@@ -465,6 +467,86 @@ public void testMap() throws Exception {
         }
     }
 
+    /**
+     * Checks deserialization of elements in the singleton map.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSingletonMap() {
+        System.setProperty(IgniteSystemProperties.IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION, "true");
+        try {
+            IgniteCache<Integer, Map<TestObject, TestObject>> c = jcache(0);
+
+            TestObject obj = new TestObject(123);
+            c.put(0, Collections.singletonMap(obj, obj));
+
+            assertEquals(1, c.get(0).size());
+
+            Map.Entry<TestObject, TestObject> entry = c.get(0).entrySet().iterator().next();
+            assertEquals(123, entry.getKey().val);
+            assertEquals(123, entry.getValue().val);
+
+            IgniteCache<Integer, Map<BinaryObject, BinaryObject>> kpc = keepBinaryCache();
+
+            Map<?, ?> cBinary = kpc.get(0);
+
+            assertEquals(Collections.singletonMap(null, null).getClass(), cBinary.getClass());
+
+            Map.Entry<?, ?> binaryEntry = kpc.get(0).entrySet().iterator().next();
+
+            assertTrue(binaryEntry.getKey() instanceof BinaryObject);
+            assertTrue(binaryEntry.getValue() instanceof BinaryObject);
+            assertEquals(Integer.valueOf(123), ((BinaryObject)binaryEntry.getKey()).field("val"));
+            assertEquals(Integer.valueOf(123), ((BinaryObject)binaryEntry.getValue()).field("val"));
+        }
+        finally {
+            System.clearProperty(IgniteSystemProperties.IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION);
+        }
+    }
+
+    /**
+     * Checks deserialization of elements in the singleton list and set.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSingletonCollection() {
+        System.setProperty(IgniteSystemProperties.IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION, "true");
+        try {
+            IgniteCache<Integer, Collection<TestObject>> c = jcache(0);
+
+            c.put(0, Collections.singletonList(new TestObject(123)));
+            c.put(1, Collections.singleton(new TestObject(123)));
+
+            assertEquals(1, c.get(0).size());
+            assertEquals(123,c.get(0).iterator().next().val);
+
+            assertEquals(1, c.get(1).size());
+            assertEquals(123,c.get(1).iterator().next().val);
+
+            IgniteCache<Integer, Collection<BinaryObject>> kpc = keepBinaryCache();
+
+            Collection<?> binaryList = kpc.get(0);
+            Collection<?> binarySet = kpc.get(1);
+
+            assertEquals(Collections.singletonList(null).getClass(), binaryList.getClass());
+            assertEquals(Collections.singleton(null).getClass(), binarySet.getClass());
+
+            assertEquals(1, binaryList.size());
+            assertEquals(1, binarySet.size());
+
+            Object obj1 = binaryList.iterator().next();
+            assertTrue(obj1 instanceof BinaryObject);
+            assertEquals(Integer.valueOf(123), ((BinaryObject)obj1).field("val"));
+
+            Object obj2 = binaryList.iterator().next();
+            assertTrue(obj2 instanceof BinaryObject);
+            assertEquals(Integer.valueOf(123), ((BinaryObject)obj2).field("val"));
+        }
+        finally {
+            System.clearProperty(IgniteSystemProperties.IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION);
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From effd6f6ced8fd34f8b7fcdeb59924605f022b0b8 Mon Sep 17 00:00:00 2001
From: pvinokurov <vinokurov.pasha@gmail.com>
Date: Wed, 7 Mar 2018 10:44:33 +0300
Subject: [PATCH 485/516] IGNITE-7718 Fixed support of the null element in
 singleton list

---
 .../ignite/internal/binary/BinaryUtils.java   |  7 ++---
 .../ignite/internal/util/IgniteUtils.java     | 27 +++++++++----------
 .../internal/util/MutableSingletonList.java   |  5 +---
 .../internal/util/MutableSingletonMap.java    |  1 -
 .../internal/util/MutableSingletonSet.java    |  1 -
 ...ridCacheBinaryObjectsAbstractSelfTest.java | 23 ++++++++++++++++
 6 files changed, 38 insertions(+), 26 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 957faa77f2b3b..35c12c416c1df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -658,11 +658,8 @@ else if (cls == ConcurrentHashMap8.class)
         else if (cls == ConcurrentHashMap.class)
             return new ConcurrentHashMap<>(U.capacity(((Map)map).size()));
 
-        if (isSingletonCollectionSerializationEnabled()) {
-            if(cls == SINGLETON_MAP_CLS){
-                return new MutableSingletonMap<>();
-            }
-        }
+        if (isSingletonCollectionSerializationEnabled() && cls == SINGLETON_MAP_CLS)
+            return new MutableSingletonMap<>();
 
         return null;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 7d055aa5317b9..b5096bf29539e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -9419,15 +9419,14 @@ public static <K, V> Map<K, V> limitedMap(int limit) {
      * Returns an immutable map if argument is singleton mutable map,
      * otherwise returns argument.
      *
-     * @param map map
-     * @param <K> type of map keys
-     * @param <V> type of map values
-     * @return argument or singleton map
+     * @param map map.
+     * @param <K> type of map keys.
+     * @param <V> type of map values.
+     * @return argument or singleton map.
      */
-    public static <K,V> Map<K,V> unwrapSingletonMap(Map<K,V> map) {
-        if(map instanceof MutableSingletonMap){
-            return ((MutableSingletonMap<K,V>)map).singletonMap();
-        }
+    public static <K, V> Map<K, V> unwrapSingletonMap(Map<K, V> map) {
+        if (map instanceof MutableSingletonMap)
+            return ((MutableSingletonMap<K, V>)map).singletonMap();
 
         return map;
     }
@@ -9436,18 +9435,16 @@ public static <K,V> Map<K,V> unwrapSingletonMap(Map<K,V> map) {
      * Returns an immutable collection if argument is singleton mutable collection,
      * otherwise returns argument.
      *
-     * @param col collection
-     * @param <T> type of collection elements
-     * @return argument or immutable singleton collection
+     * @param col collection.
+     * @param <T> type of collection elements.
+     * @return argument or immutable singleton collection.
      */
     public static <T> Collection<T> unwrapSingletonCollection(Collection<T> col) {
-        if(col instanceof MutableSingletonList){
+        if (col instanceof MutableSingletonList)
             return ((MutableSingletonList<T>)col).singletonList();
-        }
 
-        if(col instanceof MutableSingletonSet){
+        if (col instanceof MutableSingletonSet)
             return ((MutableSingletonSet<T>)col).singletonSet();
-        }
 
         return col;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonList.java
index c15974143deb9..6b6c2d7e6caec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonList.java
@@ -25,7 +25,6 @@
  * List that contains only one element. By default the list contains null element.
  */
 public class MutableSingletonList<E> extends AbstractList<E> {
-
     /** The only element of collection. */
     private E element;
 
@@ -40,6 +39,7 @@ public class MutableSingletonList<E> extends AbstractList<E> {
     /** {@inheritDoc} */
     @Override public boolean add(E e) {
         add(0, e);
+
         return true;
     }
 
@@ -63,9 +63,6 @@ public class MutableSingletonList<E> extends AbstractList<E> {
      */
     @SuppressWarnings("unchecked")
     public Collection<E> singletonList() {
-        if (element == null) {
-            throw new IllegalStateException("Singleton list is empty");
-        }
         return Collections.singletonList(element);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonMap.java b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonMap.java
index 52234e6dd2e77..e4194255e9cd3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonMap.java
@@ -26,7 +26,6 @@
  * Map that contains only one key/value. By default the map contains null/null entry.
  */
 public class MutableSingletonMap<K, V> extends AbstractMap<K, V> {
-
     /** The key of single entry. */
     protected K key;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonSet.java b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonSet.java
index 18b698b05fd42..ad1ca12dbf031 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/MutableSingletonSet.java
@@ -27,7 +27,6 @@
  * Set that contains only one element. By default the set contains null element.
  */
 public class MutableSingletonSet<E> extends AbstractSet<E> {
-
     /** The only element of set. */
     private E element;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
index dbc256eb3f797..497dae53b05e5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
@@ -479,13 +479,19 @@ public void testSingletonMap() {
 
             TestObject obj = new TestObject(123);
             c.put(0, Collections.singletonMap(obj, obj));
+            c.put(1, Collections.singletonMap(null, null));
 
             assertEquals(1, c.get(0).size());
+            assertEquals(1, c.get(1).size());
 
             Map.Entry<TestObject, TestObject> entry = c.get(0).entrySet().iterator().next();
             assertEquals(123, entry.getKey().val);
             assertEquals(123, entry.getValue().val);
 
+            Map.Entry<TestObject, TestObject> nullEntry = c.get(1).entrySet().iterator().next();
+            assertNull(nullEntry.getKey());
+            assertNull(nullEntry.getValue());
+
             IgniteCache<Integer, Map<BinaryObject, BinaryObject>> kpc = keepBinaryCache();
 
             Map<?, ?> cBinary = kpc.get(0);
@@ -496,8 +502,14 @@ public void testSingletonMap() {
 
             assertTrue(binaryEntry.getKey() instanceof BinaryObject);
             assertTrue(binaryEntry.getValue() instanceof BinaryObject);
+
             assertEquals(Integer.valueOf(123), ((BinaryObject)binaryEntry.getKey()).field("val"));
             assertEquals(Integer.valueOf(123), ((BinaryObject)binaryEntry.getValue()).field("val"));
+
+            Map.Entry<?, ?> nullBinaryEntry = kpc.get(1).entrySet().iterator().next();
+
+            assertNull(nullBinaryEntry.getKey());
+            assertNull(nullBinaryEntry.getValue());
         }
         finally {
             System.clearProperty(IgniteSystemProperties.IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION);
@@ -516,6 +528,8 @@ public void testSingletonCollection() {
 
             c.put(0, Collections.singletonList(new TestObject(123)));
             c.put(1, Collections.singleton(new TestObject(123)));
+            c.put(3, Collections.singletonList(null));
+            c.put(4, Collections.singleton(null));
 
             assertEquals(1, c.get(0).size());
             assertEquals(123,c.get(0).iterator().next().val);
@@ -523,6 +537,12 @@ public void testSingletonCollection() {
             assertEquals(1, c.get(1).size());
             assertEquals(123,c.get(1).iterator().next().val);
 
+            assertEquals(1, c.get(3).size());
+            assertNull(c.get(3).iterator().next());
+
+            assertEquals(1, c.get(4).size());
+            assertNull(c.get(4).iterator().next());
+
             IgniteCache<Integer, Collection<BinaryObject>> kpc = keepBinaryCache();
 
             Collection<?> binaryList = kpc.get(0);
@@ -541,6 +561,9 @@ public void testSingletonCollection() {
             Object obj2 = binaryList.iterator().next();
             assertTrue(obj2 instanceof BinaryObject);
             assertEquals(Integer.valueOf(123), ((BinaryObject)obj2).field("val"));
+
+            assertNull(kpc.get(3).iterator().next());
+            assertNull(kpc.get(4).iterator().next());
         }
         finally {
             System.clearProperty(IgniteSystemProperties.IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION);

From a175031ceb231d0a14979bb941f1fd065748b5ff Mon Sep 17 00:00:00 2001
From: pvinokurov <vinokurov.pasha@gmail.com>
Date: Mon, 19 Mar 2018 11:48:42 +0300
Subject: [PATCH 486/516] Fixed compilation

---
 .../binary/GridCacheBinaryObjectsAbstractSelfTest.java      | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
index 497dae53b05e5..46c3ec8da1ae6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java
@@ -479,7 +479,7 @@ public void testSingletonMap() {
 
             TestObject obj = new TestObject(123);
             c.put(0, Collections.singletonMap(obj, obj));
-            c.put(1, Collections.singletonMap(null, null));
+            c.put(1, Collections.<TestObject, TestObject>singletonMap(null, null));
 
             assertEquals(1, c.get(0).size());
             assertEquals(1, c.get(1).size());
@@ -528,8 +528,8 @@ public void testSingletonCollection() {
 
             c.put(0, Collections.singletonList(new TestObject(123)));
             c.put(1, Collections.singleton(new TestObject(123)));
-            c.put(3, Collections.singletonList(null));
-            c.put(4, Collections.singleton(null));
+            c.put(3, Collections.<TestObject>singletonList(null));
+            c.put(4, Collections.<TestObject>singleton(null));
 
             assertEquals(1, c.get(0).size());
             assertEquals(123,c.get(0).iterator().next().val);

From 6a5a4cf258b9fc06ca1b4926823695881def8374 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Tue, 3 Apr 2018 18:48:27 -0700
Subject: [PATCH 487/516] IGNITE-8153 Nodes fail to connect each other when SSL
 is enabled

---
 .../internal/util/nio/ssl/BlockingSslHandler.java      | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
index 638106f667a3c..0099c4675a381 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
@@ -373,9 +373,10 @@ private HandshakeStatus runTasks() {
      * @throws GridNioException If failed to pass event to the next filter.
      */
     private Status unwrapHandshake() throws SSLException, IgniteCheckedException {
-        // Flip input buffer so we can read the collected data.
-        readFromNet();
+        if(!inNetBuf.hasRemaining())
+            readFromNet();
 
+        // Flip input buffer so we can read the collected data.
         inNetBuf.flip();
 
         SSLEngineResult res = unwrap0();
@@ -399,7 +400,10 @@ private Status unwrapHandshake() throws SSLException, IgniteCheckedException {
         else if (res.getStatus() == BUFFER_UNDERFLOW) {
             inNetBuf.compact();
 
-            inNetBuf = expandBuffer(inNetBuf, inNetBuf.capacity() * 2);
+            if(inNetBuf.capacity() == inNetBuf.limit())
+                inNetBuf = expandBuffer(inNetBuf, inNetBuf.capacity() * 2);
+
+            readFromNet();
         }
         else
             // prepare to be written again

From ca926b139d007cd816260a8511b588ec1d77fa6a Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Mon, 9 Apr 2018 14:33:13 +0300
Subject: [PATCH 488/516] IGNITE-7904: Changed IgniteUtils::cast not to trim
 exception chains. This closes #3683.

(cherry picked from commit 3a4f23b)
---
 .../ignite/compute/ComputeTaskAdapter.java    |   2 +-
 .../processors/cache/GridCacheUtils.java      |   5 +-
 .../processors/igfs/IgfsMetaManager.java      |  30 ++-
 .../processors/job/GridJobWorker.java         |   2 +-
 .../platform/services/PlatformServices.java   |   8 +-
 .../processors/service/GridServiceProxy.java  |  27 ++-
 .../ignite/internal/util/IgniteUtils.java     |  32 +--
 .../IgniteComputeResultExceptionTest.java     | 186 ++++++++++++++++++
 .../GridCacheAbstractFullApiSelfTest.java     |   9 +-
 .../closure/GridClosureSerializationTest.java |   2 +-
 .../GridServiceProcessorProxySelfTest.java    |  12 +-
 .../IgniteComputeGridTestSuite.java           |   2 +
 ...teCacheLockPartitionOnAffinityRunTest.java |  46 ++---
 .../CacheJdbcPojoStoreFactorySelfTest.java    |  11 +-
 .../GridServiceInjectionSelfTest.java         |  64 +++---
 .../GridSpringResourceInjectionSelfTest.java  |  60 +++---
 16 files changed, 337 insertions(+), 161 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/compute/ComputeTaskAdapter.java b/modules/core/src/main/java/org/apache/ignite/compute/ComputeTaskAdapter.java
index c5352aacb41ff..fc55ad9f30f8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/compute/ComputeTaskAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/compute/ComputeTaskAdapter.java
@@ -99,7 +99,7 @@ public abstract class ComputeTaskAdapter<T, R> implements ComputeTask<T, R> {
                 return ComputeJobResultPolicy.FAILOVER;
 
             throw new IgniteException("Remote job threw user exception (override or implement ComputeTask.result(..) " +
-                "method if you would like to have automatic failover for this exception).", e);
+                "method if you would like to have automatic failover for this exception): " + e.getMessage(), e);
         }
 
         // Wait for all job responses.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index a78a9226b065a..f68a1354c3613 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1437,8 +1437,9 @@ else if (e instanceof CacheAtomicUpdateTimeoutCheckedException)
         else if (e instanceof ClusterTopologyServerNotFoundException)
             return new CacheServerNotFoundException(e.getMessage(), e);
 
-        if (e.getCause() instanceof CacheException)
-            return (CacheException)e.getCause();
+        CacheException ce = X.cause(e, CacheException.class);
+        if (ce != null)
+            return ce;
 
         if (e.getCause() instanceof NullPointerException)
             return (NullPointerException)e.getCause();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index ffa502b0b75ea..744a2c1e8b9f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -69,6 +69,7 @@
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T1;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.transactions.TransactionConcurrency;
@@ -248,29 +249,20 @@ boolean isClient() {
      * @return Result.
      */
     <T> T runClientTask(IgfsClientAbstractCallable<T> task) {
-        try {
-            return runClientTask(IgfsUtils.ROOT_ID, task);
-        }
-        catch (ClusterTopologyException e) {
-            throw new IgfsException("Failed to execute operation because there are no IGFS metadata nodes." , e);
-        }
-    }
-
-    /**
-     * Run client task.
-     *
-     * @param affinityFileId Affinity fileId.
-     * @param task Task.
-     * @return Result.
-     */
-    <T> T runClientTask(IgniteUuid affinityFileId, IgfsClientAbstractCallable<T> task) {
         try {
             return (cfg.isColocateMetadata()) ?
-                clientCompute().affinityCall(cfg.getMetaCacheName(), affinityFileId, task) :
+                clientCompute().affinityCall(cfg.getMetaCacheName(), IgfsUtils.ROOT_ID, task) :
                 clientCompute().call(task);
         }
-        catch (ClusterTopologyException e) {
-            throw new IgfsException("Failed to execute operation because there are no IGFS metadata nodes." , e);
+        catch (Exception e) {
+            if (X.hasCause(e, ClusterTopologyException.class))
+                throw new IgfsException("Failed to execute operation because there are no IGFS metadata nodes." , e);
+
+            IgfsException igfsEx = X.cause(e, IgfsException.class);
+            if (igfsEx != null)
+                throw igfsEx;
+
+            throw e;
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index acefde72eefab..8697cd1e4be87 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -672,7 +672,7 @@ else if (sysStopping && X.hasCause(e, InterruptedException.class, IgniteInterrup
 
         if (msg == null) {
             msg = "Failed to execute job due to unexpected runtime exception [jobId=" + ses.getJobId() +
-                ", ses=" + ses + ']';
+                ", ses=" + ses + ", err=" + e.getMessage() + ']';
 
             ex = new ComputeUserUndeclaredException(msg, e);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
index 12aab746a0b41..4ca592cde7ac1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.internal.processors.platform.utils.PlatformWriterClosure;
 import org.apache.ignite.internal.processors.service.GridServiceProxy;
 import org.apache.ignite.internal.util.future.IgniteFutureImpl;
+import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.T3;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.services.Service;
@@ -479,7 +480,12 @@ public Object invoke(String mthdName, boolean srvKeepBinary, Object[] args)
 
                 Method mtd = getMethod(serviceClass, mthdName, args);
 
-                return ((GridServiceProxy)proxy).invokeMethod(mtd, args);
+                try {
+                    return ((GridServiceProxy)proxy).invokeMethod(mtd, args);
+                }
+                catch (Throwable t) {
+                    throw IgniteUtils.cast(t);
+                }
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index 3a40b907f91b2..f809dba3e7264 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -46,6 +46,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
@@ -150,7 +151,7 @@ private boolean hasLocalNode(ClusterGroup prj) {
      * @return Result.
      */
     @SuppressWarnings("BusyWait")
-    public Object invokeMethod(final Method mtd, final Object[] args) {
+    public Object invokeMethod(final Method mtd, final Object[] args) throws Throwable {
         if (U.isHashCodeMethod(mtd))
             return System.identityHashCode(proxy);
         else if (U.isEqualsMethod(mtd))
@@ -205,6 +206,12 @@ else if (U.isToStringMethod(mtd))
                     throw e;
                 }
                 catch (IgniteCheckedException e) {
+                    // Rethrow original service method exception so that calling user code can handle it correctly.
+                    ServiceProxyException svcProxyE = X.cause(e, ServiceProxyException.class);
+
+                    if (svcProxyE != null)
+                        throw svcProxyE.getCause();
+
                     throw U.convertException(e);
                 }
                 catch (Exception e) {
@@ -352,7 +359,7 @@ private class ProxyInvocationHandler implements InvocationHandler {
 
         /** {@inheritDoc} */
         @SuppressWarnings("BusyWait")
-        @Override public Object invoke(Object proxy, final Method mtd, final Object[] args) {
+        @Override public Object invoke(Object proxy, final Method mtd, final Object[] args) throws Throwable {
             return invokeMethod(mtd, args);
         }
     }
@@ -418,8 +425,7 @@ private ServiceProxyCallable(String mtdName, String svcName, Class[] argTypes, O
                 return mtd.invoke(svcCtx.service(), args);
             }
             catch (InvocationTargetException e) {
-                // Get error message.
-                throw new IgniteCheckedException(e.getCause().getMessage(), e);
+                throw new ServiceProxyException(e.getCause());
             }
         }
 
@@ -444,4 +450,17 @@ private ServiceProxyCallable(String mtdName, String svcName, Class[] argTypes, O
             return S.toString(ServiceProxyCallable.class, this);
         }
     }
+
+    /**
+     * Exception class that wraps an exception thrown by the service implementation.
+     */
+    private static class ServiceProxyException extends RuntimeException {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        ServiceProxyException(Throwable cause) {
+            super(cause);
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index b5096bf29539e..5c176c9c7383d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -7217,35 +7217,23 @@ public static Exception unwrap(Throwable t) {
     }
 
     /**
-     * Casts this throwable as {@link IgniteCheckedException}. Creates wrapping
-     * {@link IgniteCheckedException}, if needed.
+     * Casts the passed {@code Throwable t} to {@link IgniteCheckedException}.<br>
+     * If {@code t} is a {@link GridClosureException}, it is unwrapped and then cast to {@link IgniteCheckedException}.
+     * If {@code t} is an {@link IgniteCheckedException}, it is returned.
+     * If {@code t} is not a {@link IgniteCheckedException}, a new {@link IgniteCheckedException} caused by {@code t}
+     * is returned.
      *
      * @param t Throwable to cast.
-     * @return Grid exception.
+     * @return {@code t} cast to {@link IgniteCheckedException}.
      */
     public static IgniteCheckedException cast(Throwable t) {
         assert t != null;
 
-        while (true) {
-            if (t instanceof Error)
-                throw (Error)t;
+        t = unwrap(t);
 
-            if (t instanceof GridClosureException) {
-                t = ((GridClosureException)t).unwrap();
-
-                continue;
-            }
-
-            if (t instanceof IgniteCheckedException)
-                return (IgniteCheckedException)t;
-
-            if (!(t instanceof IgniteException) || t.getCause() == null)
-                return new IgniteCheckedException(t);
-
-            assert t.getCause() != null; // ...and it is IgniteException.
-
-            t = t.getCause();
-        }
+        return t instanceof IgniteCheckedException
+            ? (IgniteCheckedException)t
+            : new IgniteCheckedException(t);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java
new file mode 100644
index 0000000000000..fab5de6d25229
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
+import org.apache.ignite.compute.ComputeTask;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Testing that if {@link ComputeTask#result(ComputeJobResult, List)} throws an {@link IgniteException}
+ * then that exception is thrown as the execution result.
+ */
+public class IgniteComputeResultExceptionTest extends GridCommonAbstractTest {
+    /** */
+    public void testIgniteExceptionExecute() throws Exception {
+        checkExecuteException(new IgniteException());
+    }
+
+    /** */
+    public void testIgniteExceptionWithCauseExecute() throws Exception {
+        checkExecuteException(new IgniteException(new Exception()));
+    }
+
+    /** */
+    public void testIgniteExceptionWithCauseChainExecute() throws Exception {
+        checkExecuteException(new IgniteException(new Exception(new Throwable())));
+    }
+
+    /** */
+    public void testCustomExceptionExecute() throws Exception {
+        checkExecuteException(new TaskException());
+    }
+
+    /** */
+    public void testCustomExceptionWithCauseExecute() throws Exception {
+        checkExecuteException(new TaskException(new Exception()));
+    }
+
+    /** */
+    public void testCustomExceptionWithCauseChainExecute() throws Exception {
+        checkExecuteException(new TaskException(new Exception(new Throwable())));
+    }
+
+    /** */
+    private void checkExecuteException(IgniteException resE) throws Exception {
+        try (Ignite ignite = startGrid()) {
+            IgniteCompute compute = ignite.compute();
+            try {
+                compute.execute(new ResultExceptionTask(resE), null);
+            } catch (IgniteException e) {
+                assertSame(resE, e);
+            }
+        }
+    }
+
+    /** */
+    public void testIgniteExceptionExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new IgniteException());
+    }
+
+    /** */
+    public void testIgniteExceptionWithCauseExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new IgniteException(new Exception()));
+    }
+
+    /** */
+    public void testIgniteExceptionWithCauseChainExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new IgniteException(new Exception(new Throwable())));
+    }
+
+
+    /** */
+    public void testCustomExceptionExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new TaskException());
+    }
+
+    /** */
+    public void testCustomExceptionWithCauseExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new TaskException(new Exception()));
+    }
+
+    /** */
+    public void testCustomExceptionWithCauseChainExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new TaskException(new Exception(new Throwable())));
+    }
+
+    /** */
+    private void checkExecuteAsyncException(IgniteException resE) throws Exception {
+        try (Ignite ignite = startGrid()) {
+            IgniteCompute compute = ignite.compute();
+            ComputeTaskFuture<Object> fut = compute.executeAsync(new ResultExceptionTask(resE), null);
+            try {
+                fut.get();
+            } catch (IgniteException e) {
+                assertSame(resE, e);
+            }
+        }
+    }
+
+    /** */
+    private static class TaskException extends IgniteException {
+        /** */
+        public TaskException() {
+            // No-op.
+        }
+
+        /** */
+        public TaskException(Throwable cause) {
+            super(cause);
+        }
+    }
+
+    /** */
+    private static class NoopJob implements ComputeJob {
+        /** */
+        @Override public void cancel() {
+            // No-op.
+        }
+
+        /** */
+        @Override public Object execute() throws IgniteException {
+            return null;
+        }
+    }
+
+    /** */
+    private static class ResultExceptionTask implements ComputeTask<Object, Object> {
+        /** */
+        private final IgniteException resE;
+
+        /**
+         * @param resE Exception to be rethrown by the
+         */
+        ResultExceptionTask(IgniteException resE) {
+            this.resE = resE;
+        }
+
+        /** */
+        @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
+            @Nullable Object arg) throws IgniteException {
+            Map<ComputeJob, ClusterNode> jobs = new HashMap<>();
+
+            for (ClusterNode node : subgrid)
+                jobs.put(new NoopJob(), node);
+
+            return jobs;
+        }
+
+        /** */
+        @Override
+        public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) throws IgniteException {
+            throw resE;
+        }
+
+        /** */
+        @Nullable @Override public Object reduce(List<ComputeJobResult> results) throws IgniteException {
+            return null;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
index 6ec06398afd77..b14b543ddf1f5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
@@ -83,6 +83,7 @@
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -4611,12 +4612,16 @@ private void waitForIteratorsCleared(IgniteCache<String, Integer> cache, int sec
 
                 checkIteratorsCleared();
             }
-            catch (AssertionFailedError e) {
+            catch (Throwable t) {
+                // If AssertionFailedError is in the chain, assume we need to wait and retry.
+                if (!X.hasCause(t, AssertionFailedError.class))
+                    throw t;
+                
                 if (i == 9) {
                     for (int j = 0; j < gridCount(); j++)
                         executeOnLocalOrRemoteJvm(j, new PrintIteratorStateTask());
 
-                    throw e;
+                    throw t;
                 }
 
                 log.info("Iterators not cleared, will wait");
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
index 2426dd7bdbc4e..d9fb35af093fe 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
@@ -68,7 +68,7 @@ public void testSerializationFailure() throws Exception {
         final IgniteEx ignite0 = grid(0);
         final IgniteEx ignite1 = grid(1);
 
-        GridTestUtils.assertThrows(null, new Callable<Object>() {
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
                 ignite1.compute(ignite1.cluster().forNode(ignite0.localNode())).call(new IgniteCallable<Object>() {
                     @Override public Object call() throws Exception {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
index d1c5294d48604..97d5f05e9e52b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
@@ -89,7 +89,7 @@ public void testException() throws Exception {
 
                 return null;
             }
-        }, IgniteException.class, "Test exception");
+        }, ErrorServiceException.class, "Test exception");
 
     }
 
@@ -450,9 +450,15 @@ protected class ErrorServiceImpl implements ErrorService {
 
         /** {@inheritDoc} */
         @Override public void go() throws Exception {
-            throw new Exception("Test exception");
+            throw new ErrorServiceException("Test exception");
         }
     }
 
-
+    /** */
+    private static class ErrorServiceException extends Exception {
+        /** */
+        ErrorServiceException(String msg) {
+            super(msg);
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 390a90c95756e..9ef4fe740b5b0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.GridTaskTimeoutSelfTest;
 import org.apache.ignite.internal.IgniteComputeEmptyClusterGroupTest;
 import org.apache.ignite.internal.IgniteComputeJobOneThreadTest;
+import org.apache.ignite.internal.IgniteComputeResultExceptionTest;
 import org.apache.ignite.internal.IgniteComputeTopologyExceptionTest;
 import org.apache.ignite.internal.IgniteExecutorServiceTest;
 import org.apache.ignite.internal.IgniteExplicitImplicitDeploymentSelfTest;
@@ -153,6 +154,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridMultinodeRedeployIsolatedModeSelfTest.class);
         suite.addTestSuite(IgniteComputeEmptyClusterGroupTest.class);
         suite.addTestSuite(IgniteComputeTopologyExceptionTest.class);
+        suite.addTestSuite(IgniteComputeResultExceptionTest.class);
         suite.addTestSuite(GridTaskFailoverAffinityRunTest.class);
         suite.addTestSuite(TaskNodeRestartTest.class);
         suite.addTestSuite(IgniteRoundRobinErrorAfterClientReconnectTest.class);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
index fbc94ede1e3a3..37a7479eb00e1 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
@@ -23,6 +23,7 @@
 import java.io.ObjectOutput;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
@@ -362,39 +363,20 @@ public void testMultipleCaches() throws Exception {
     public void testCheckReservePartitionException() throws Exception {
         int orgId = primaryKey(grid(1).cache(Organization.class.getSimpleName()));
 
-        try {
-            grid(0).compute().affinityRun(
-                Arrays.asList(Organization.class.getSimpleName(), OTHER_CACHE_NAME),
-                new Integer(orgId),
-                new IgniteRunnable() {
-                    @Override public void run() {
-                        // No-op.
-                    }
-                });
-
-            fail("Exception is expected");
-        }
-        catch (Exception e) {
-            assertTrue(e.getMessage()
-                .startsWith("Failed partition reservation. Partition is not primary on the node."));
-        }
-
-        try {
-            grid(0).compute().affinityCall(
-                Arrays.asList(Organization.class.getSimpleName(), OTHER_CACHE_NAME),
-                new Integer(orgId),
-                new IgniteCallable<Object>() {
-                    @Override public Object call() throws Exception {
-                        return null;
-                    }
-                });
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                grid(0).compute().affinityRun(
+                    Arrays.asList(Organization.class.getSimpleName(), OTHER_CACHE_NAME),
+                    new Integer(orgId),
+                    new IgniteRunnable() {
+                        @Override public void run() {
+                            // No-op.
+                        }
+                    });
 
-            fail("Exception is expected");
-        }
-        catch (Exception e) {
-            assertTrue(e.getMessage()
-                .startsWith("Failed partition reservation. Partition is not primary on the node."));
-        }
+                return null;
+            }
+        }, IgniteException.class, "Failed partition reservation. Partition is not primary on the node.");
     }
 
     /**
diff --git a/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java b/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java
index e3549355d6e3b..d13a2c7b2f925 100644
--- a/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java
@@ -21,7 +21,7 @@
 import java.util.concurrent.Callable;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect;
 import org.apache.ignite.cache.store.jdbc.dialect.JdbcDialect;
@@ -69,15 +69,14 @@ public void testSerializable() throws Exception {
      * @throws Exception If failed.
      */
     public void testIncorrectBeanConfiguration() throws Exception {
-        GridTestUtils.assertThrows(log, new Callable<Object>() {
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
-                try(Ignite ignite = Ignition.start("modules/spring/src/test/config/pojo-incorrect-store-cache.xml")) {
-                    ignite.cache(CACHE_NAME).getConfiguration(CacheConfiguration.class).
-                        getCacheStoreFactory().create();
+                try (Ignite ignored = Ignition.start("modules/spring/src/test/config/pojo-incorrect-store-cache.xml")) {
+                    // No-op.
                 }
                 return null;
             }
-        }, IgniteException.class, "Failed to load bean in application context");
+        }, IgniteCheckedException.class, "Failed to load bean in application context");
     }
 
     /**
diff --git a/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridServiceInjectionSelfTest.java b/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridServiceInjectionSelfTest.java
index b59bf2457f4de..52581b6d699ac 100644
--- a/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridServiceInjectionSelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridServiceInjectionSelfTest.java
@@ -18,11 +18,13 @@
 package org.apache.ignite.internal.processors.resource;
 
 import java.io.Serializable;
-import org.apache.ignite.IgniteException;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.resources.ServiceResource;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -124,23 +126,22 @@ public void testClosureFieldLocalProxy() throws Exception {
      * @throws Exception If failed.
      */
     public void testClosureFieldWithIncorrectType() throws Exception {
-        try {
-            grid(0).compute().call(new IgniteCallable<Object>() {
-                @ServiceResource(serviceName = SERVICE_NAME1)
-                private String svcName;
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Void>() {
+            @Override public Void call() {
+                grid(0).compute().call(new IgniteCallable<Object>() {
+                    @ServiceResource(serviceName = SERVICE_NAME1)
+                    private String svcName;
 
-                @Override public Object call() throws Exception {
-                    fail();
+                    @Override public Object call() throws Exception {
+                        fail();
 
-                    return null;
-                }
-            });
+                        return null;
+                    }
+                });
 
-            fail();
-        }
-        catch (IgniteException e) {
-            assertTrue(e.getMessage().startsWith("Resource field is not assignable from the resource"));
-        }
+                return null;
+            }
+        }, IgniteCheckedException.class, "Resource field is not assignable from the resource");
     }
 
     /**
@@ -221,23 +222,22 @@ private void service(DummyService svc) {
      * @throws Exception If failed.
      */
     public void testClosureMethodWithIncorrectType() throws Exception {
-        try {
-            grid(0).compute().call(new IgniteCallable<Object>() {
-                @ServiceResource(serviceName = SERVICE_NAME1)
-                private void service(String svcs) {
-                    fail();
-                }
-
-                @Override public Object call() throws Exception {
-                    return null;
-                }
-            });
-
-            fail();
-        }
-        catch (IgniteException e) {
-            assertTrue(e.getMessage().startsWith("Setter does not have single parameter of required type"));
-        }
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Void>() {
+            @Override public Void call() {
+                grid(0).compute().call(new IgniteCallable<Object>() {
+                    @ServiceResource(serviceName = SERVICE_NAME1)
+                    private void service(String svcs) {
+                        fail();
+                    }
+
+                    @Override public Object call() throws Exception {
+                        return null;
+                    }
+                });
+
+                return null;
+            }
+        }, IgniteCheckedException.class, "Setter does not have single parameter of required type");
     }
 
     /**
diff --git a/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridSpringResourceInjectionSelfTest.java b/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridSpringResourceInjectionSelfTest.java
index b989ac80819c4..5d5f146a01b9b 100644
--- a/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridSpringResourceInjectionSelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridSpringResourceInjectionSelfTest.java
@@ -27,6 +27,8 @@
 import org.apache.ignite.resources.SpringResource;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.springframework.beans.factory.NoSuchBeanDefinitionException;
+import org.springframework.beans.factory.NoUniqueBeanDefinitionException;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 
 /**
@@ -93,7 +95,7 @@ public void testClosureFieldByResourceClassWithMultipleBeans() throws Exception
         Ignite anotherGrid = IgniteSpring.start(anotherCfg, new ClassPathXmlApplicationContext(
             "/org/apache/ignite/internal/processors/resource/spring-resource-with-duplicate-beans.xml"));
 
-        Throwable err = assertError(new IgniteCallable<Object>() {
+        assertError(new IgniteCallable<Object>() {
             @SpringResource(resourceClass = DummyResourceBean.class)
             private transient DummyResourceBean dummyRsrcBean;
 
@@ -102,11 +104,9 @@ public void testClosureFieldByResourceClassWithMultipleBeans() throws Exception
 
                 return null;
             }
-        }, anotherGrid, null);
-
-        assertTrue("Unexpected message: " + err.getMessage(), err.getMessage().startsWith("No qualifying bean of type " +
-            "[org.apache.ignite.internal.processors.resource.GridSpringResourceInjectionSelfTest$DummyResourceBean]" +
-            " is defined: expected single matching bean but found 2:"));
+        }, anotherGrid, NoUniqueBeanDefinitionException.class, "No qualifying bean of type " +
+            "'org.apache.ignite.internal.processors.resource.GridSpringResourceInjectionSelfTest$DummyResourceBean'" +
+            " available: expected single matching bean but found 2:");
 
         G.stop("anotherGrid", false);
     }
@@ -124,7 +124,7 @@ public void testClosureFieldWithWrongResourceName() {
 
                 return null;
             }
-        }, "No bean named 'nonExistentResource' is defined");
+        }, grid, NoSuchBeanDefinitionException.class, "No bean named 'nonExistentResource' defined");
     }
 
     /**
@@ -140,7 +140,7 @@ public void testClosureFieldWithWrongResourceClass() {
 
                 return null;
             }
-        }, "No qualifying bean of type [org.apache.ignite.internal.processors.resource." +
+        }, grid, NoSuchBeanDefinitionException.class, "No qualifying bean of type [org.apache.ignite.internal.processors.resource." +
             "GridSpringResourceInjectionSelfTest$AnotherDummyResourceBean] is defined");
     }
 
@@ -157,7 +157,7 @@ public void testClosureFieldByResourceClassAndName() {
 
                 return null;
             }
-        }, "Either bean name or its class must be specified in @SpringResource, but not both");
+        }, grid, IgniteException.class, "Either bean name or its class must be specified in @SpringResource, but not both");
     }
 
     /**
@@ -173,7 +173,7 @@ public void testClosureFieldWithNoParams() {
 
                 return null;
             }
-        }, "Either bean name or its class must be specified in @SpringResource, but not both");
+        }, grid, IgniteException.class, "Either bean name or its class must be specified in @SpringResource, but not both");
     }
 
     /**
@@ -232,7 +232,7 @@ public void testClosureMethodWithResourceClassWithMultipleBeans() throws Excepti
             "/org/apache/ignite/internal/processors/resource/spring-resource-with-duplicate-beans.xml"));
 
         try {
-            Throwable err = assertError(new IgniteCallable<Object>() {
+            assertError(new IgniteCallable<Object>() {
                 private DummyResourceBean dummyRsrcBean;
 
                 @SpringResource(resourceClass = DummyResourceBean.class)
@@ -247,11 +247,9 @@ private void setDummyResourceBean(DummyResourceBean dummyRsrcBean) {
 
                     return null;
                 }
-            }, anotherGrid, null);
-
-            assertTrue("Unexpected message: " + err.getMessage(), err.getMessage().startsWith("No qualifying bean of type " +
+            }, anotherGrid, NoUniqueBeanDefinitionException.class, "No qualifying bean of type " +
                 "[org.apache.ignite.internal.processors.resource.GridSpringResourceInjectionSelfTest$DummyResourceBean]" +
-                " is defined: expected single matching bean but found 2:"));
+                " is defined: expected single matching bean but found 2:");
         }
         finally {
             G.stop("anotherGrid", false);
@@ -275,7 +273,7 @@ private void setDummyResourceBean(DummyResourceBean dummyRsrcBean) {
 
                 return null;
             }
-        }, "No bean named 'nonExistentResource' is defined");
+        }, grid, NoSuchBeanDefinitionException.class, "No bean named 'nonExistentResource' is defined");
     }
 
     /**
@@ -295,7 +293,7 @@ private void setDummyResourceBean(AnotherDummyResourceBean dummyRsrcBean) {
 
                 return null;
             }
-        }, "No qualifying bean of type [org.apache.ignite.internal.processors.resource" +
+        }, grid, NoSuchBeanDefinitionException.class,"No qualifying bean of type [org.apache.ignite.internal.processors.resource" +
                 ".GridSpringResourceInjectionSelfTest$AnotherDummyResourceBean] is defined");
     }
 
@@ -312,7 +310,7 @@ public void testClosureMethodByResourceClassAndName() {
 
                 return null;
             }
-        }, "Either bean name or its class must be specified in @SpringResource, but not both");
+        }, grid, IgniteException.class, "Either bean name or its class must be specified in @SpringResource, but not both");
     }
 
     /**
@@ -328,32 +326,24 @@ public void testClosureMethodWithNoParams() {
 
                 return null;
             }
-        }, "Either bean name or its class must be specified in @SpringResource, but not both");
+        }, grid, IgniteException.class, "Either bean name or its class must be specified in @SpringResource, but not both");
     }
 
     /**
-     * @param job {@link IgniteCallable} to be run
-     * @param grid Node.
-     * @param expEMsg Message that {@link IgniteException} thrown from <tt>job</tt> should bear
-     * @return Thrown error.
+     * @param job {@link IgniteCallable} to be run.
+     * @param grid Node to run the job on.
+     * @param expE Expected exception type.
+     * @param expEMsg Expected exception message.
      */
     @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    private Throwable assertError(final IgniteCallable<?> job, final Ignite grid, String expEMsg) {
-        return GridTestUtils.assertThrows(log, new Callable<Object>() {
+    private void assertError(final IgniteCallable<?> job, final Ignite grid, Class<? extends Throwable> expE,
+        String expEMsg) {
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
                 grid.compute(grid.cluster().forLocal()).call(job);
                 return null;
             }
-        }, IgniteException.class, expEMsg);
-    }
-
-    /**
-     * @param job {@link IgniteCallable} to be run
-     * @param expEMsg Message that {@link IgniteException} thrown from <tt>job</tt> should bear
-     */
-    @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    private void assertError(final IgniteCallable<?> job, String expEMsg) {
-        assertError(job, grid, expEMsg);
+        }, expE, expEMsg);
     }
 
     /**

From a80afeb509297b5ee871c43ee87592c8493608f0 Mon Sep 17 00:00:00 2001
From: Roman Guseinov <gromcase@gmail.com>
Date: Mon, 9 Apr 2018 14:45:44 +0300
Subject: [PATCH 489/516] IGNITE-7944: Disconnected client node tries to send
 JOB_CANCEL message. Applied fix: - Skip sending message if client
 disconnected; - Throw IgniteCheckedException if a client node is disconnected
 and communication client is null. This closes #3737.

(cherry picked from commit d70477b)
---
 .../processors/task/GridTaskProcessor.java    |   2 +-
 .../tcp/TcpCommunicationSpi.java              |  26 +-
 ...cpCommunicationSpiSkipMessageSendTest.java | 414 ++++++++++++++++++
 .../IgniteSpiCommunicationSelfTestSuite.java  |   3 +
 4 files changed, 442 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 995a25618a89b..1ecb04dca4ef8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -167,7 +167,7 @@ public GridTaskProcessor(GridKernalContext ctx) {
         IgniteClientDisconnectedCheckedException err = disconnectedError(reconnectFut);
 
         for (GridTaskWorker<?, ?> worker : tasks.values())
-            worker.finishTask(null, err);
+            worker.finishTask(null, err, false);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 1432b51e8454b..7e8a0c4233389 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -63,6 +63,7 @@
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.util.GridConcurrentFactory;
@@ -2409,6 +2410,11 @@ private void sendMessage0(ClusterNode node, Message msg, IgniteInClosure<IgniteE
         if (log.isTraceEnabled())
             log.trace("Sending message with ack to node [node=" + node + ", msg=" + msg + ']');
 
+        if (isLocalNodeDisconnected()) {
+            throw new IgniteSpiException("Failed to send a message to remote node because local node has " +
+                "been disconnected [rmtNodeId=" + node.id() + ']');
+        }
+
         ClusterNode locNode = getLocalNode();
 
         if (locNode == null)
@@ -2463,6 +2469,18 @@ private void sendMessage0(ClusterNode node, Message msg, IgniteInClosure<IgniteE
         }
     }
 
+    /**
+     * @return {@code True} if local node in disconnected state.
+     */
+    private boolean isLocalNodeDisconnected() {
+        boolean disconnected = false;
+
+        if (ignite instanceof IgniteKernal)
+            disconnected = ((IgniteKernal)ignite).context().clientDisconnected();
+
+        return disconnected;
+    }
+
     /**
      * @param nodeId Node ID.
      * @param rmvClient Client to remove.
@@ -2608,8 +2626,12 @@ private GridCommunicationClient reserveClient(ClusterNode node, int connIdx) thr
 
                 client = fut.get();
 
-                if (client == null)
-                    continue;
+                if (client == null) {
+                    if (isLocalNodeDisconnected())
+                        throw new IgniteCheckedException("Unable to create TCP client due to local node disconnecting.");
+                    else
+                        continue;
+                }
 
                 if (getSpiContext().node(nodeId) == null) {
                     if (removeNodeClient(nodeId, client))
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java
new file mode 100644
index 0000000000000..c4bc8f2922767
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.Socket;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import org.apache.ignite.spi.collision.fifoqueue.FifoQueueCollisionSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Tests that the client will be segmented in time and won't hang due to canceling compute jobs.
+ */
+public class TcpCommunicationSpiSkipMessageSendTest extends GridCommonAbstractTest {
+    /** */
+    private static final CountDownLatch COMPUTE_JOB_STARTED = new CountDownLatch(1);
+
+    /** */
+    private static final long FAILURE_DETECTION_TIMEOUT = 10000;
+
+    /** */
+    private static final long JOIN_TIMEOUT = 10000;
+
+    /** */
+    private static final long START_JOB_TIMEOUT = 10000;
+
+    /** */
+    private static final long DISABLE_NETWORK_DELAY = 2000;
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 2 * 60 * 1000;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientSegmented() throws Exception {
+        Ignite server = null;
+        Ignite client = null;
+
+        try {
+            server = Ignition.start(getConfig(false));
+
+            final CountDownLatch clientDisconnected = new CountDownLatch(1);
+            final CountDownLatch clientSegmented = new CountDownLatch(1);
+
+            client = startClient(clientDisconnected, clientSegmented);
+
+            final IgniteCompute compute = client.compute();
+
+            runJobAsync(compute);
+
+            if (!COMPUTE_JOB_STARTED.await(START_JOB_TIMEOUT, TimeUnit.MILLISECONDS))
+                fail("Compute job wasn't started.");
+
+            disableNetwork(client);
+
+            if (!clientDisconnected.await(FAILURE_DETECTION_TIMEOUT * 3, TimeUnit.MILLISECONDS))
+                fail("Client wasn't disconnected.");
+
+            if (!clientSegmented.await(JOIN_TIMEOUT * 2, TimeUnit.MILLISECONDS))
+                fail("Client wasn't segmented.");
+        }
+        finally {
+            if (client != null)
+                client.close();
+
+            if (server != null)
+                server.close();
+        }
+    }
+
+    /**
+     * Simulate network disabling.
+     *
+     * @param ignite Ignite instance.
+     * @throws IgniteInterruptedCheckedException If thread sleep interrupted.
+     * @throws InterruptedException If waiting for network disabled failed (interrupted).
+     */
+    private void disableNetwork(Ignite ignite) throws IgniteInterruptedCheckedException, InterruptedException {
+        U.sleep(DISABLE_NETWORK_DELAY);
+
+        CustomCommunicationSpi communicationSpi = (CustomCommunicationSpi)ignite.configuration().getCommunicationSpi();
+
+        CustomDiscoverySpi discoverySpi = (CustomDiscoverySpi)ignite.configuration().getDiscoverySpi();
+
+        discoverySpi.disableNetwork();
+
+        communicationSpi.disableNetwork();
+
+        if (!discoverySpi.awaitNetworkDisabled(FAILURE_DETECTION_TIMEOUT * 2))
+            fail("Network wasn't disabled.");
+    }
+
+    /**
+     * Start compute jobs in the separate thread.
+     *
+     * @param compute Ignite compute instance.
+     */
+    private void runJobAsync(final IgniteCompute compute) {
+        new Thread(new Runnable() {
+            @Override public void run() {
+                try {
+                    compute.call(new IgniteCallable<Integer>() {
+                        @Override public Integer call() throws Exception {
+                            COMPUTE_JOB_STARTED.countDown();
+
+                            // Simulate long-running job.
+                            new CountDownLatch(1).await();
+
+                            return null;
+                        }
+                    });
+                }
+                catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }).start();
+    }
+
+    /**
+     * Create Communication Spi instance.
+     *
+     * @param client Is a client node.
+     * @return Communication Spi.
+     */
+    private TcpCommunicationSpi getCommunicationSpi(boolean client) {
+        TcpCommunicationSpi spi = new CustomCommunicationSpi(client);
+
+        spi.setName("CustomCommunicationSpi");
+
+        return spi;
+    }
+
+    /**
+     * Create Discovery Spi instance.
+     *
+     * @return Discovery Spi.
+     */
+    private TcpDiscoverySpi getDiscoverySpi() {
+        TcpDiscoverySpi spi = new CustomDiscoverySpi();
+
+        spi.setName("CustomDiscoverySpi");
+
+        spi.setIpFinder(LOCAL_IP_FINDER);
+
+        return spi;
+    }
+
+    /**
+     * Create Ignite configuration.
+     *
+     * @param clientMode Client mode.
+     * @return Ignite configuration.
+     */
+    private IgniteConfiguration getConfig(boolean clientMode) {
+        IgniteConfiguration cfg = new IgniteConfiguration();
+
+        cfg.setIgniteInstanceName(clientMode ? "client-node" : "server-node");
+
+        cfg.setClientMode(clientMode);
+
+        cfg.setCommunicationSpi(getCommunicationSpi(clientMode));
+
+        if (!clientMode) {
+            cfg.setDiscoverySpi(getDiscoverySpi());
+
+            FifoQueueCollisionSpi collisionSpi = new FifoQueueCollisionSpi();
+
+            collisionSpi.setParallelJobsNumber(1);
+
+            cfg.setCollisionSpi(collisionSpi);
+        }
+        else {
+            cfg.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT);
+
+            cfg.setDiscoverySpi(getDiscoverySpi().setJoinTimeout(JOIN_TIMEOUT));
+        }
+
+        return cfg;
+    }
+
+    /**
+     * Start client node.
+     *
+     * @param clientDisconnected Client is disconnected.
+     * @param clientSegmented Client is segmented.
+     * @return Ignite instance.
+     */
+    private Ignite startClient(final CountDownLatch clientDisconnected, final CountDownLatch clientSegmented) {
+        Ignite ignite = Ignition.start(getConfig(true));
+
+        IgnitePredicate<Event> locLsnr = new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event event) {
+                log.info("Client node received event: " + event.name());
+
+                if (event.type() == EventType.EVT_CLIENT_NODE_DISCONNECTED)
+                    clientDisconnected.countDown();
+
+                if (event.type() == EventType.EVT_NODE_SEGMENTED)
+                    clientSegmented.countDown();
+
+                return true;
+            }
+        };
+
+        ignite.events().localListen(locLsnr,
+            EventType.EVT_NODE_SEGMENTED,
+            EventType.EVT_CLIENT_NODE_DISCONNECTED);
+
+        return ignite;
+    }
+
+    /**
+     * Communication Spi that emulates connection troubles.
+     */
+    class CustomCommunicationSpi extends TcpCommunicationSpi {
+        /** Network is disabled. */
+        private volatile boolean networkDisabled = false;
+
+        /** Additional logging is enabled. */
+        private final boolean logEnabled;
+
+        /**
+         * @param enableLogs Enable additional logging.
+         */
+        CustomCommunicationSpi(boolean enableLogs) {
+            super();
+            this.logEnabled = enableLogs;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            String message = msg.toString();
+
+            if (logEnabled)
+                log.info("CustomCommunicationSpi.sendMessage: " + message);
+
+            if (message.contains("TOPIC_JOB_CANCEL"))
+                closeTcpConnections();
+
+            super.sendMessage(node, msg, ackC);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
+            if (logEnabled)
+                log.info(String.format("CustomCommunicationSpi.createTcpClient [networkDisabled=%s, node=%s]", networkDisabled, node));
+
+            if (networkDisabled) {
+                IgniteSpiOperationTimeoutHelper timeoutHelper = new IgniteSpiOperationTimeoutHelper(this, !node.isClient());
+
+                long timeout = timeoutHelper.nextTimeoutChunk(getConnectTimeout());
+
+                if (logEnabled)
+                    log.info("CustomCommunicationSpi.createTcpClient [timeoutHelper.nextTimeoutChunk=" + timeout + "]");
+
+                sleep(timeout);
+
+                return null;
+            }
+            else
+                return super.createTcpClient(node, connIdx);
+        }
+
+        /**
+         * Simulate network disabling.
+         */
+        void disableNetwork() {
+            networkDisabled = true;
+        }
+
+        /**
+         * Close communication clients. It will lead that sendMessage method will be trying to create new ones.
+         */
+        private void closeTcpConnections() {
+            final ConcurrentMap<UUID, GridCommunicationClient[]> clients = U.field(this, "clients");
+
+            Set<UUID> ids = clients.keySet();
+
+            if (ids.size() > 0) {
+                log.info("Close TCP clients: " + ids);
+
+                for (UUID nodeId : ids) {
+                    GridCommunicationClient[] clients0 = clients.remove(nodeId);
+
+                    if (clients0 != null) {
+                        for (GridCommunicationClient client : clients0) {
+                            if (client != null)
+                                client.forceClose();
+                        }
+                    }
+                }
+
+                log.info("TCP clients are closed.");
+            }
+        }
+    }
+
+    /**
+     * Discovery Spi that emulates connection troubles.
+     */
+    class CustomDiscoverySpi extends TcpDiscoverySpi {
+        /** Network is disabled. */
+        private volatile boolean networkDisabled = false;
+
+        /** */
+        private final CountDownLatch networkDisabledLatch = new CountDownLatch(1);
+
+        /** */
+        CustomDiscoverySpi() {
+            super();
+
+            setName("CustomDiscoverySpi");
+        }
+
+        /** {@inheritDoc} */
+        @Override protected <T> T readMessage(Socket sock, @Nullable InputStream in,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (networkDisabled) {
+                sleep(timeout);
+
+                return null;
+            }
+            else
+                return super.readMessage(sock, in, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (networkDisabled) {
+                sleep(timeout);
+
+                networkDisabledLatch.countDown();
+
+                throw new IgniteCheckedException("CustomDiscoverySpi: network is disabled.");
+            }
+            else
+                super.writeToSocket(sock, msg, timeout);
+        }
+
+        /**
+         * Simulate network disabling.
+         */
+        void disableNetwork() {
+            networkDisabled = true;
+        }
+
+        /**
+         * Wait until the network is disabled.
+         */
+        boolean awaitNetworkDisabled(long timeout) throws InterruptedException {
+            return networkDisabledLatch.await(timeout, TimeUnit.MILLISECONDS);
+        }
+    }
+
+    /**
+     * Sleeps for given number of milliseconds.
+     *
+     * @param timeout Time to sleep (2000 ms by default).
+     * @throws IgniteInterruptedCheckedException If current thread interrupted.
+     */
+    static void sleep(long timeout) throws IgniteInterruptedCheckedException {
+        if (timeout > 0)
+            U.sleep(timeout);
+        else
+            U.sleep(2000);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
index 8e96a3f223604..2bc8c589ceadb 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiDropNodesTest;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiFaultyClientTest;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiHalfOpenedConnectionTest;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiSkipMessageSendTest;
 
 /**
  * Test suite for all communication SPIs.
@@ -77,6 +78,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTest(new TestSuite(GridTcpCommunicationSpiConfigSelfTest.class));
 
+        suite.addTest(new TestSuite(TcpCommunicationSpiSkipMessageSendTest.class));
+
         suite.addTest(new TestSuite(TcpCommunicationSpiFaultyClientTest.class));
         suite.addTest(new TestSuite(TcpCommunicationSpiDropNodesTest.class));
         suite.addTest(new TestSuite(TcpCommunicationSpiHalfOpenedConnectionTest.class));

From d9bb7420474ab3276feb381be9d41069fa7b9247 Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Mon, 9 Apr 2018 18:49:35 +0300
Subject: [PATCH 490/516] Fixed compilation.

---
 .../ignite/internal/IgniteComputeResultExceptionTest.java    | 5 +++--
 .../tcp/TcpCommunicationSpiSkipMessageSendTest.java          | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java
index fab5de6d25229..bcc0b0d698018 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java
@@ -113,8 +113,9 @@ public void testCustomExceptionWithCauseChainExecuteAsync() throws Exception {
     /** */
     private void checkExecuteAsyncException(IgniteException resE) throws Exception {
         try (Ignite ignite = startGrid()) {
-            IgniteCompute compute = ignite.compute();
-            ComputeTaskFuture<Object> fut = compute.executeAsync(new ResultExceptionTask(resE), null);
+            IgniteCompute compute = ignite.compute().withAsync();
+            compute.execute(new ResultExceptionTask(resE), null);
+            ComputeTaskFuture<Object> fut = compute.future();
             try {
                 fut.get();
             } catch (IgniteException e) {
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java
index c4bc8f2922767..98dfa795b27b4 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java
@@ -199,7 +199,7 @@ private TcpDiscoverySpi getDiscoverySpi() {
     private IgniteConfiguration getConfig(boolean clientMode) {
         IgniteConfiguration cfg = new IgniteConfiguration();
 
-        cfg.setIgniteInstanceName(clientMode ? "client-node" : "server-node");
+        cfg.setGridName(clientMode ? "client-node" : "server-node");
 
         cfg.setClientMode(clientMode);
 
@@ -292,7 +292,7 @@ class CustomCommunicationSpi extends TcpCommunicationSpi {
                 log.info(String.format("CustomCommunicationSpi.createTcpClient [networkDisabled=%s, node=%s]", networkDisabled, node));
 
             if (networkDisabled) {
-                IgniteSpiOperationTimeoutHelper timeoutHelper = new IgniteSpiOperationTimeoutHelper(this, !node.isClient());
+                IgniteSpiOperationTimeoutHelper timeoutHelper = new IgniteSpiOperationTimeoutHelper(this);
 
                 long timeout = timeoutHelper.nextTimeoutChunk(getConnectTimeout());
 

From 7dff4dfae8ae88734fce0a3fcc901a2326fe06c6 Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Mon, 9 Apr 2018 20:05:22 +0300
Subject: [PATCH 491/516] Fixed Java 7 compilation.

---
 .../cache/IgniteCacheLockPartitionOnAffinityRunTest.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
index 37a7479eb00e1..cc656363e785a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
@@ -361,7 +361,7 @@ public void testMultipleCaches() throws Exception {
      * @throws Exception If failed.
      */
     public void testCheckReservePartitionException() throws Exception {
-        int orgId = primaryKey(grid(1).cache(Organization.class.getSimpleName()));
+        final int orgId = primaryKey(grid(1).cache(Organization.class.getSimpleName()));
 
         GridTestUtils.assertThrowsAnyCause(log, new Callable<Void>() {
             @Override public Void call() throws Exception {

From 9660af67b4d83da146d517b97bee6138d8e24022 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 25 Apr 2018 13:36:27 +0300
Subject: [PATCH 492/516] GG-13317: Fixed DR hanging on node stop.

(cherry picked from commit b5ba5da)
---
 .../distributed/dht/GridDhtCacheEntry.java    |  7 +++-
 .../util/future/GridCompoundFuture.java       | 37 +++++++++++++++----
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
index 44720500de105..d7bef598fb4c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
@@ -41,6 +41,7 @@
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.lang.GridPlainRunnable;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -70,6 +71,7 @@ public class GridDhtCacheEntry extends GridDistributedCacheEntry {
     private volatile ReaderId[] rdrs = ReaderId.EMPTY_ARRAY;
 
     /** Local partition. */
+    @GridToStringExclude
     private final GridDhtLocalPartition locPart;
 
     /**
@@ -720,7 +722,10 @@ protected String cacheName() {
 
     /** {@inheritDoc} */
     @Override public synchronized String toString() {
-        return S.toString(GridDhtCacheEntry.class, this, "super", super.toString());
+        return S.toString(GridDhtCacheEntry.class, this,
+            "part", locPart.id(),
+            "super", super.toString());
+
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
index 0e9575fe0d70d..7f8c0effdc987 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridCompoundFuture.java
@@ -18,8 +18,10 @@
 package org.apache.ignite.internal.util.future;
 
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.IgniteFutureCancelledCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -97,13 +99,13 @@ public GridCompoundFuture(@Nullable IgniteReducer<T, R> rdc) {
                     onDone(rdc.reduce());
             }
             catch (RuntimeException e) {
-                U.error(null, "Failed to execute compound future reducer: " + this, e);
+                logError(null, "Failed to execute compound future reducer: " + this, e);
 
                 // Exception in reducer is a bug, so we bypass checkComplete here.
                 onDone(e);
             }
             catch (AssertionError e) {
-                U.error(null, "Failed to execute compound future reducer: " + this, e);
+                logError(null, "Failed to execute compound future reducer: " + this, e);
 
                 // Bypass checkComplete because need to rethrow.
                 onDone(e);
@@ -118,18 +120,21 @@ public GridCompoundFuture(@Nullable IgniteReducer<T, R> rdc) {
         }
         catch (IgniteCheckedException e) {
             if (!processFailure(e, fut)) {
-                U.error(null, "Failed to execute compound future reducer: " + this, e);
+                if (e instanceof NodeStoppingException)
+                    logDebug(logger(), "Failed to execute compound future reducer, node stopped.");
+                else
+                    logError(null, "Failed to execute compound future reducer: " + this, e);
 
                 onDone(e);
             }
         }
         catch (RuntimeException e) {
-            U.error(null, "Failed to execute compound future reducer: " + this, e);
+            logError(null, "Failed to execute compound future reducer: " + this, e);
 
             onDone(e);
         }
         catch (AssertionError e) {
-            U.error(null, "Failed to execute compound future reducer: " + this, e);
+            logError(null, "Failed to execute compound future reducer: " + this, e);
 
             // Bypass checkComplete because need to rethrow.
             onDone(e);
@@ -287,12 +292,12 @@ private void checkComplete() {
                 onDone(rdc != null ? rdc.reduce() : null);
             }
             catch (RuntimeException e) {
-                U.error(null, "Failed to execute compound future reducer: " + this, e);
+                logError(null, "Failed to execute compound future reducer: " + this, e);
 
                 onDone(e);
             }
             catch (AssertionError e) {
-                U.error(null, "Failed to execute compound future reducer: " + this, e);
+                logError(null, "Failed to execute compound future reducer: " + this, e);
 
                 onDone(e);
 
@@ -301,6 +306,24 @@ private void checkComplete() {
         }
     }
 
+    /**
+     * @param log IgniteLogger.
+     * @param msg ShortMessage.
+     * @param e Exception.
+     */
+    protected void logError(IgniteLogger log, String msg, Throwable e) {
+        U.error(log, msg, e);
+    }
+
+    /**
+     * @param log IgniteLogger.
+     * @param msg ShortMessage.
+     */
+    protected void logDebug(IgniteLogger log, String msg) {
+        if (log != null && log.isDebugEnabled())
+            log.debug(msg);
+    }
+
     /**
      * Returns future at the specified position in this list.
      *

From 39ad5cdfb3a77540c351f8021d393eb158b87811 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 25 Apr 2018 13:36:43 +0300
Subject: [PATCH 493/516] Fix javadoc test

(cherry picked from commit 345e84d)
---
 .../cassandra/serializer/package-info.java    | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 modules/cassandra/serializers/src/main/java/org/apache/ignite/cache/store/cassandra/serializer/package-info.java

diff --git a/modules/cassandra/serializers/src/main/java/org/apache/ignite/cache/store/cassandra/serializer/package-info.java b/modules/cassandra/serializers/src/main/java/org/apache/ignite/cache/store/cassandra/serializer/package-info.java
new file mode 100644
index 0000000000000..cad12b7646c38
--- /dev/null
+++ b/modules/cassandra/serializers/src/main/java/org/apache/ignite/cache/store/cassandra/serializer/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains Cassandra serializers.
+ */
+package org.apache.ignite.cache.store.cassandra.serializer;
\ No newline at end of file

From 84bc16eaa0f54395e2334df49f0493289b58a55e Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 26 Apr 2018 16:24:50 +0300
Subject: [PATCH 494/516] IGNITE-5910: Method stopGrid(name) doesn't work in
 multiJvm mode - Fixes #2382.

Signed-off-by: Nikolay Izhikov <nizhikov@apache.org>

(cherry picked from commit 5ccb548)
---
 .../junits/GridAbstractTest.java              |  7 +--
 .../junits/multijvm/IgniteProcessProxy.java   | 48 ++++++++++++++++---
 2 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 22c796f6cd3e1..f649e1ec98cbc 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -880,12 +880,13 @@ protected void stopGrid(@Nullable String gridName, boolean cancel) {
     @SuppressWarnings({"deprecation"})
     protected void stopGrid(@Nullable String gridName, boolean cancel, boolean awaitTop) {
         try {
-            Ignite ignite = grid(gridName);
+            IgniteEx ignite = grid(gridName);
 
             assert ignite != null : "Ignite returned null grid for name: " + gridName;
 
-            info(">>> Stopping grid [name=" + ignite.name() + ", id=" +
-                ((IgniteKernal)ignite).context().localNodeId() + ']');
+            UUID id = ignite instanceof IgniteProcessProxy ? ignite.localNode().id() : ignite.context().localNodeId();
+
+            info(">>> Stopping grid [name=" + ignite.name() + ", id=" + id + ']');
 
             if (!isRemoteJvm(gridName))
                 G.stop(gridName, cancel);
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
index 7c622084b233a..c6425fea714ce 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
@@ -39,15 +39,16 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteFileSystem;
 import org.apache.ignite.IgniteIllegalStateException;
+import org.apache.ignite.IgniteLock;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.IgniteQueue;
-import org.apache.ignite.IgniteLock;
 import org.apache.ignite.IgniteScheduler;
 import org.apache.ignite.IgniteSemaphore;
 import org.apache.ignite.IgniteServices;
 import org.apache.ignite.IgniteSet;
 import org.apache.ignite.IgniteTransactions;
+import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterNode;
@@ -94,6 +95,9 @@ public class IgniteProcessProxy implements IgniteEx {
     /** Property that specify alternative {@code JAVA_HOME}. */
     private static final String TEST_MULTIJVM_JAVA_HOME = "test.multijvm.java.home";
 
+    /** Waiting milliseconds of the left of a node to topology. */
+    private static final long NODE_LEFT_TIMEOUT = 30_000L;
+
     /** Jvm process with ignite instance. */
     private final transient GridJavaProcess proc;
 
@@ -217,12 +221,39 @@ public static IgniteProcessProxy ignite(String gridName) {
      *
      * @param gridName Grid name.
      * @param cancel If {@code true} then all jobs currently will be cancelled.
+     * @throws Exception In case of the node stopping error.
      */
-    public static void stop(String gridName, boolean cancel) {
-        IgniteProcessProxy proxy = gridProxies.get(gridName);
+    public static void stop(String gridName, boolean cancel) throws Exception {
+        final IgniteProcessProxy proxy = gridProxies.get(gridName);
 
         if (proxy != null) {
-            proxy.remoteCompute().run(new StopGridTask(gridName, cancel));
+            final CountDownLatch rmtNodeStoppedLatch = new CountDownLatch(1);
+            final UUID rmNodeId = proxy.getId();
+
+            proxy.locJvmGrid.events().localListen(new IgnitePredicateX<Event>() {
+                @Override public boolean applyx(Event e) {
+                    if (((DiscoveryEvent)e).eventNode().id().equals(rmNodeId)) {
+                        rmtNodeStoppedLatch.countDown();
+
+                        return false;
+                    }
+
+                    return true;
+                }
+            }, EventType.EVT_NODE_LEFT);
+
+            try {
+                proxy.remoteCompute().withAsync().run(new StopGridTask(gridName, cancel));
+
+                if (!rmtNodeStoppedLatch.await(NODE_LEFT_TIMEOUT, TimeUnit.MILLISECONDS))
+                    throw new IllegalStateException("Remote node has not stopped [id=" + rmNodeId + ']');
+            }
+            catch (Throwable t) {
+                proxy.log().error("Failed to stop grid [igniteInstanceName=" + gridName +
+                    ", cancel=" + cancel + ']', t);
+
+                throw t;
+            }
 
             gridProxies.remove(gridName, proxy);
         }
@@ -670,7 +701,8 @@ public IgniteCompute remoteCompute() {
     }
 
     /**
-     *
+     * Executes {@link Ignition#stop(String, boolean)} with given arguments in a separated thread, doesn't wait up the
+     * fulfillment.
      */
     private static class StopGridTask implements IgniteRunnable {
         /** Grid name. */
@@ -690,7 +722,11 @@ public StopGridTask(String gridName, boolean cancel) {
 
         /** {@inheritDoc} */
         @Override public void run() {
-            G.stop(gridName, cancel);
+            new Thread(new Runnable() {
+                @Override public void run() {
+                    G.stop(gridName, cancel);
+                }
+            }).start();
         }
     }
 

From b8e3db39ca905d2c6ed13779c69f90537f08f5c3 Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Tue, 22 May 2018 16:43:45 +0300
Subject: [PATCH 495/516] IGNITE-8501 Retry on GridServiceNotFoundException in
 GridServiceProxy needs to be fixed - Fixes #4004.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 7c2fe64)
---
 .../processors/service/GridServiceProxy.java  | 22 +++++--
 ...idServiceProxyClientReconnectSelfTest.java | 62 ++++++++++++++++++-
 2 files changed, 78 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index f809dba3e7264..8834ab487deac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -206,13 +206,25 @@ else if (U.isToStringMethod(mtd))
                     throw e;
                 }
                 catch (IgniteCheckedException e) {
-                    // Rethrow original service method exception so that calling user code can handle it correctly.
-                    ServiceProxyException svcProxyE = X.cause(e, ServiceProxyException.class);
+                    // Check if ignorable exceptions are in the cause chain.
+                    Throwable ignorableCause = X.cause(e, GridServiceNotFoundException.class);
 
-                    if (svcProxyE != null)
-                        throw svcProxyE.getCause();
+                    if (ignorableCause == null)
+                        ignorableCause = X.cause(e, ClusterTopologyCheckedException.class);
 
-                    throw U.convertException(e);
+                    if (ignorableCause != null) {
+                        if (log.isDebugEnabled())
+                            log.debug("Service was not found or topology changed (will retry): " + ignorableCause.getMessage());
+                    }
+                    else {
+                        // Rethrow original service method exception so that calling user code can handle it correctly.
+                        ServiceProxyException svcProxyE = X.cause(e, ServiceProxyException.class);
+
+                        if (svcProxyE != null)
+                            throw svcProxyE.getCause();
+
+                        throw U.convertException(e);
+                    }
                 }
                 catch (Exception e) {
                     throw new IgniteException(e);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProxyClientReconnectSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProxyClientReconnectSelfTest.java
index 1fa6b0f00ee16..72783744d16bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProxyClientReconnectSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProxyClientReconnectSelfTest.java
@@ -82,13 +82,49 @@ public void testClientReconnect() throws Exception {
 
         startGrid("server");
 
-        assert latch.await(10, TimeUnit.SECONDS);
+        assertTrue(latch.await(10, TimeUnit.SECONDS));
 
         client.services().deployClusterSingleton("my-service", new MyServiceImpl());
 
         assertEquals(42, proxy.hello());
     }
 
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientReconnectLongServiceInit() throws Exception {
+        startGrid("server");
+
+        Ignite client = startGrid("client");
+
+        client.services().deployClusterSingleton("my-service", new MyLongInitServiceImpl());
+
+        MyService proxy = client.services().serviceProxy("my-service", MyService.class, false);
+
+        assertEquals(9001, proxy.hello());
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        client.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event event) {
+                latch.countDown();
+
+                return true;
+            }
+        }, EventType.EVT_CLIENT_NODE_RECONNECTED);
+
+        stopGrid("server");
+
+        startGrid("server");
+
+        assertTrue(latch.await(10, TimeUnit.SECONDS));
+
+        client.services().deployClusterSingleton("my-service", new MyLongInitServiceImpl());
+
+        assertEquals(9001, proxy.hello());
+    }
+
     /**
      */
     private interface MyService extends Service {
@@ -121,4 +157,28 @@ private static class MyServiceImpl implements MyService {
             // No-op.
         }
     }
+
+    /**
+     */
+    private static class MyLongInitServiceImpl implements MyService {
+        /** {@inheritDoc} */
+        @Override public int hello() {
+            return 9001;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(ServiceContext ctx) throws Exception {
+            Thread.sleep(5_000);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+            // No-op.
+        }
+    }
 }

From 392ca42695e52620d0d6a5a264780880bb8b5b77 Mon Sep 17 00:00:00 2001
From: NSAmelchev <nsamelchev@gmail.com>
Date: Mon, 14 May 2018 14:57:21 +0300
Subject: [PATCH 496/516] IGNITE-5087 Enum comparison fails after
 marshal-unmarshal with BinaryMarshaller.

Cherry-picked from 01f504ff83cc77f80d37981b5c5a15b653861bbd
---
 .../ignite/internal/binary/BinaryUtils.java   | 19 ++++-
 .../internal/binary/BinaryWriterExImpl.java   |  4 +-
 .../builder/BinaryBuilderSerializer.java      |  6 +-
 .../internal/binary/BinaryEnumsSelfTest.java  | 73 +++++++++++++++++++
 .../binary/BinaryMarshallerSelfTest.java      | 64 +++++++++++++++-
 5 files changed, 158 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 35c12c416c1df..92d81efa12ff1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -586,7 +586,7 @@ public static byte typeByClass(Class<?> cls) {
         if (type != null)
             return type;
 
-        if (cls.isEnum())
+        if (isEnum(cls))
             return GridBinaryMarshaller.ENUM;
 
         if (cls.isArray())
@@ -1110,7 +1110,7 @@ else if (isSpecialCollection(cls))
             return BinaryWriteMode.COL;
         else if (isSpecialMap(cls))
             return BinaryWriteMode.MAP;
-        else if (cls.isEnum())
+        else if (isEnum(cls))
             return BinaryWriteMode.ENUM;
         else if (cls == Class.class)
             return BinaryWriteMode.CLASS;
@@ -1141,6 +1141,21 @@ public static boolean isSpecialMap(Class cls) {
         return HashMap.class.equals(cls) || LinkedHashMap.class.equals(cls) || BinaryUtils.isSingletonMap(cls);
     }
 
+    /**
+     * Check if class represents a Enum.
+     *
+     * @param cls Class.
+     * @return {@code True} if this is a Enum class.
+     */
+    public static boolean isEnum(Class cls) {
+        if (cls.isEnum())
+            return true;
+
+        Class sCls = cls.getSuperclass();
+
+        return sCls != null && sCls.isEnum();
+    }
+
     /**
      * @return Value.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index 3289780e3fd0e..d49f9ff474727 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -783,7 +783,7 @@ void doWriteEnum(@Nullable Enum<?> val) {
         if (val == null)
             out.writeByte(GridBinaryMarshaller.NULL);
         else {
-            BinaryClassDescriptor desc = ctx.descriptorForClass(val.getClass(), false);
+            BinaryClassDescriptor desc = ctx.descriptorForClass(val.getDeclaringClass(), false);
 
             out.unsafeEnsure(1 + 4);
 
@@ -793,7 +793,7 @@ void doWriteEnum(@Nullable Enum<?> val) {
                 out.unsafeWriteInt(desc.typeId());
             else {
                 out.unsafeWriteInt(GridBinaryMarshaller.UNREGISTERED_TYPE_ID);
-                doWriteString(val.getClass().getName());
+                doWriteString(val.getDeclaringClass().getName());
             }
 
             out.writeInt(val.ordinal());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
index 6974176f90053..75b84835fd5ad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
@@ -109,8 +109,8 @@ public void writeValue(BinaryWriterExImpl writer, Object val, boolean forceCol,
             return;
         }
 
-        if (val.getClass().isEnum()) {
-            String clsName = val.getClass().getName();
+        if (BinaryUtils.isEnum(val.getClass())) {
+            String clsName = ((Enum)val).getDeclaringClass().getName();
 
             int typeId = writer.context().typeId(clsName);
             String typeName = writer.context().userTypeName(clsName);
@@ -120,7 +120,7 @@ public void writeValue(BinaryWriterExImpl writer, Object val, boolean forceCol,
             writer.context().updateMetadata(typeId, meta);
 
             // Need register class for marshaller to be able to deserialize enum value.
-            writer.context().descriptorForClass(val.getClass(), false);
+            writer.context().descriptorForClass(((Enum)val).getDeclaringClass(), false);
 
             writer.writeByte(GridBinaryMarshaller.ENUM);
             writer.writeInt(typeId);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryEnumsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryEnumsSelfTest.java
index 91add0de06d39..3fd6121efd50a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryEnumsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryEnumsSelfTest.java
@@ -406,6 +406,44 @@ public void testZeroTypeId() throws Exception {
         assert enumObj.type().isEnum();
     }
 
+    /**
+     * Test operations on simple type with declared body which is registered in advance.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDeclaredBodyEnumRegistered() throws Exception {
+        checkDeclaredBodyEnum(true);
+    }
+
+    /**
+     * Test operations on simple type with declared body which is not registered in advance.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDeclaredBodyEnumNotRegistered() throws Exception {
+        checkDeclaredBodyEnum(false);
+    }
+
+    /**
+     * Check enums with declared body.
+     *
+     * @param registered Registered flag.
+     * @throws Exception If failed.
+     */
+    private void checkDeclaredBodyEnum(boolean registered) throws Exception {
+        startUp(registered);
+
+        cache1.put(1, DeclaredBodyEnum.ONE);
+
+        if (registered) {
+            assertEquals(DeclaredBodyEnum.ONE, cache1.get(1));
+            assertEquals(DeclaredBodyEnum.ONE, cache2.get(1));
+        }
+
+        validate((BinaryObject) cacheBinary1.get(1), DeclaredBodyEnum.ONE);
+        validate((BinaryObject) cacheBinary2.get(1), DeclaredBodyEnum.ONE);
+    }
+
     /**
      * Validate simple array.
      *
@@ -470,6 +508,23 @@ private void validate(BinaryObject obj, EnumType val) {
         assertEquals(node2.binary().typeId(EnumType.class.getName()), obj.type().typeId());
 
         assertEquals(val.ordinal(), obj.enumOrdinal());
+
+        //if (register)
+        //    assertEquals(val.name(), obj.enumName());
+    }
+    /**
+     * Validate single value.
+     *
+     * @param obj Binary value.
+     * @param val Expected value.
+     */
+    private void validate(BinaryObject obj, DeclaredBodyEnum val) {
+        assertTrue(obj.type().isEnum());
+
+        assertEquals(node1.binary().typeId(DeclaredBodyEnum.class.getName()), obj.type().typeId());
+        assertEquals(node2.binary().typeId(DeclaredBodyEnum.class.getName()), obj.type().typeId());
+
+        assertEquals(val.ordinal(), obj.enumOrdinal());
     }
 
     /**
@@ -504,4 +559,22 @@ public enum EnumType {
         ONE,
         TWO
     }
+
+    /**
+     * Enumeration for tests.
+     */
+    public enum DeclaredBodyEnum {
+        ONE {
+            @Override boolean isSupported() {
+                return false;
+            }
+        },
+        TWO {
+            @Override boolean isSupported() {
+                return false;
+            }
+        };
+
+        abstract boolean isSupported();
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index 31d6f31eef961..cfb2c6bf5276e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -614,6 +614,33 @@ public void testEnum() throws Exception {
         assertEquals(TestEnum.B, marshalUnmarshal(TestEnum.B, marsh));
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeclaredBodyEnum() throws Exception {
+        final MarshallerContextTestImpl ctx = new MarshallerContextTestImpl() {{
+            this.registerClassName(1, EnumObject.class.getName());
+            this.registerClassName(2, DeclaredBodyEnum.class.getName());
+        }};
+
+        BinaryMarshaller marsh = binaryMarshaller();
+        marsh.setContext(ctx);
+
+        EnumObject obj = new EnumObject(1L, "test 1", DeclaredBodyEnum.TWO);
+
+        final byte[] marshal = marsh.marshal(obj);
+        final Object restored = marsh.unmarshal(marshal, null);
+
+        assertTrue(restored instanceof EnumObject);
+
+        obj = (EnumObject)restored;
+
+        assertEquals(1, obj.id);
+        assertEquals(DeclaredBodyEnum.TWO.ordinal(), obj.type.ordinal());
+        assertEquals(DeclaredBodyEnum.TWO, obj.type);
+        assertTrue(obj.type == DeclaredBodyEnum.TWO);
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -5347,4 +5374,39 @@ public ObjectRaw(int val0, int val1) {
             val1 = reader.rawReader().readInt();
         }
     }
-}
\ No newline at end of file
+
+    /** */
+    private static class EnumObject implements Serializable {
+        /** */
+        private long id;
+
+        /** */
+        private String name;
+
+        /** */
+        private DeclaredBodyEnum type;
+
+        /** */
+        EnumObject(final long id, final String name, final DeclaredBodyEnum type) {
+            this.id = id;
+            this.name = name;
+            this.type = type;
+        }
+    }
+
+    /** */
+    public enum DeclaredBodyEnum {
+        ONE {
+            @Override boolean isSupported() {
+                return false;
+            }
+        },
+        TWO {
+            @Override boolean isSupported() {
+                return false;
+            }
+        };
+
+        abstract boolean isSupported();
+    }
+}

From 02a16f3355f319418463c91e729b44bcad9a7539 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Fri, 25 May 2018 14:23:59 -0700
Subject: [PATCH 497/516] IGNITE-8547 - Use JVM serialization for enum values
 with OptimizedMarshaller, avoid deadlock.

Cherry-picked from 5564a14f7cb.
---
 .../ignite/internal/binary/BinaryUtils.java   |  19 +-
 .../builder/BinaryBuilderSerializer.java      |   3 +-
 .../ignite/internal/util/IgniteUtils.java     |  15 ++
 .../OptimizedObjectOutputStream.java          |   4 +-
 .../MarshallerEnumDeadlockMultiJvmTest.java   | 189 ++++++++++++++++++
 .../IgniteMarshallerSelfTestSuite.java        |   2 +
 6 files changed, 213 insertions(+), 19 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerEnumDeadlockMultiJvmTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 92d81efa12ff1..7beb9b3ba8338 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -586,7 +586,7 @@ public static byte typeByClass(Class<?> cls) {
         if (type != null)
             return type;
 
-        if (isEnum(cls))
+        if (U.isEnum(cls))
             return GridBinaryMarshaller.ENUM;
 
         if (cls.isArray())
@@ -1110,7 +1110,7 @@ else if (isSpecialCollection(cls))
             return BinaryWriteMode.COL;
         else if (isSpecialMap(cls))
             return BinaryWriteMode.MAP;
-        else if (isEnum(cls))
+        else if (U.isEnum(cls))
             return BinaryWriteMode.ENUM;
         else if (cls == Class.class)
             return BinaryWriteMode.CLASS;
@@ -1141,21 +1141,6 @@ public static boolean isSpecialMap(Class cls) {
         return HashMap.class.equals(cls) || LinkedHashMap.class.equals(cls) || BinaryUtils.isSingletonMap(cls);
     }
 
-    /**
-     * Check if class represents a Enum.
-     *
-     * @param cls Class.
-     * @return {@code True} if this is a Enum class.
-     */
-    public static boolean isEnum(Class cls) {
-        if (cls.isEnum())
-            return true;
-
-        Class sCls = cls.getSuperclass();
-
-        return sCls != null && sCls.isEnum();
-    }
-
     /**
      * @return Value.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
index 75b84835fd5ad..0a396266b4d30 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.binary.BinaryUtils;
 import org.apache.ignite.internal.binary.BinaryWriterExImpl;
 import org.apache.ignite.internal.binary.GridBinaryMarshaller;
+import org.apache.ignite.internal.util.IgniteUtils;
 
 /**
  *
@@ -109,7 +110,7 @@ public void writeValue(BinaryWriterExImpl writer, Object val, boolean forceCol,
             return;
         }
 
-        if (BinaryUtils.isEnum(val.getClass())) {
+        if (IgniteUtils.isEnum(val.getClass())) {
             String clsName = ((Enum)val).getDeclaringClass().getName();
 
             int typeId = writer.context().typeId(clsName);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 5c176c9c7383d..4cbf4f660e2ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -6119,6 +6119,21 @@ public static boolean isJdk(Class<?> cls) {
         return s.startsWith("java.") || s.startsWith("javax.");
     }
 
+    /**
+     * Check if given class represents a Enum.
+     *
+     * @param cls Class to check.
+     * @return {@code True} if this is a Enum class.
+     */
+    public static boolean isEnum(Class cls) {
+        if (cls.isEnum())
+            return true;
+
+        Class sCls = cls.getSuperclass();
+
+        return sCls != null && sCls.isEnum();
+    }
+
     /**
      * Converts {@link InterruptedException} to {@link IgniteCheckedException}.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
index 98d85a0a0ec1b..6cb5c31b5ce58 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
@@ -42,6 +42,7 @@
 import org.apache.ignite.internal.util.GridHandleTable;
 import org.apache.ignite.internal.util.io.GridDataOutput;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.marshaller.MarshallerContext;
 
@@ -166,7 +167,8 @@ private void writeObject0(Object obj) throws IOException {
         if (obj == null)
             writeByte(NULL);
         else {
-            if (obj instanceof Throwable && !(obj instanceof Externalizable)) {
+            if (obj instanceof Throwable && !(obj instanceof Externalizable) || U.isEnum(obj.getClass())) {
+                // Avoid problems with differing Enum objects or Enum implementation class deadlocks.
                 writeByte(JDK);
 
                 try {
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerEnumDeadlockMultiJvmTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerEnumDeadlockMultiJvmTest.java
new file mode 100644
index 0000000000000..9f267d7c38263
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerEnumDeadlockMultiJvmTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.marshaller;
+
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import javax.cache.configuration.Factory;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Contains test of Enum marshalling with various {@link Marshaller}s. See IGNITE-8547 for details.
+ */
+public class MarshallerEnumDeadlockMultiJvmTest extends GridCommonAbstractTest {
+    /** */
+    private Factory<Marshaller> marshFactory;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String instanceName) throws Exception {
+        return super.getConfiguration(instanceName).setMarshaller(marshFactory.create());
+    }
+
+    /** */
+    public void testJdkMarshaller() throws Exception {
+        marshFactory = new JdkMarshallerFactory();
+
+        runRemoteUnmarshal();
+    }
+
+    /** */
+    public void testOptimizedMarshaller() throws Exception {
+        marshFactory = new OptimizedMarshallerFactory();
+
+        runRemoteUnmarshal();
+    }
+
+    /** */
+    public void testBinaryMarshaller() throws Exception {
+        marshFactory = new BinaryMarshallerFactory();
+
+        runRemoteUnmarshal();
+    }
+
+    /** */
+    private void runRemoteUnmarshal() throws Exception {
+        Ignite ignite = startGrid(0);
+
+        byte[] one = ignite.configuration().getMarshaller().marshal(DeclaredBodyEnum.ONE);
+        byte[] two = ignite.configuration().getMarshaller().marshal(DeclaredBodyEnum.TWO);
+
+        startGrid(1);
+
+        ignite.compute(ignite.cluster().forRemotes()).call(new UnmarshalCallable(one, two));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean isMultiJvm() {
+        return true;
+    }
+
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** */
+    private static class OptimizedMarshallerFactory implements Factory<Marshaller> {
+        /** {@inheritDoc} */
+        @Override public Marshaller create() {
+            return new OptimizedMarshaller(false);
+        }
+    }
+
+    /** */
+    private static class BinaryMarshallerFactory implements Factory<Marshaller> {
+        /** {@inheritDoc} */
+        @Override public Marshaller create() {
+            return new BinaryMarshaller();
+        }
+    }
+
+    /** */
+    private static class JdkMarshallerFactory implements Factory<Marshaller> {
+        /** {@inheritDoc} */
+        @Override public Marshaller create() {
+            return new JdkMarshaller();
+        }
+    }
+
+    /**
+     * Attempts to unmarshal both in-built and inner-class enum values at exactly the same time in multiple threads.
+     */
+    private static class UnmarshalCallable implements IgniteCallable<Object> {
+        /** */
+        private final byte[] one;
+        /** */
+        private final byte[] two;
+        /** */
+        @IgniteInstanceResource
+        private Ignite ign;
+
+        /** */
+        public UnmarshalCallable(byte[] one, byte[] two) {
+            this.one = one;
+            this.two = two;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object call() throws Exception {
+            ExecutorService executor = Executors.newFixedThreadPool(2);
+
+            final CyclicBarrier start = new CyclicBarrier(2);
+
+            for (int i = 0; i < 2; i++) {
+                final int ii = i;
+
+                executor.execute(new Runnable() {
+                    @Override public void run() {
+                        try {
+                            start.await();
+
+                            if (ii == 0)
+                                ign.configuration().getMarshaller().unmarshal(one, null);
+                            else
+                                ign.configuration().getMarshaller().unmarshal(two, null);
+                        }
+                        catch (Exception e) {
+                            e.printStackTrace();
+                        }
+                    }
+                });
+            }
+
+            try {
+                executor.shutdown();
+
+                executor.awaitTermination(5, TimeUnit.SECONDS);
+
+                if (!executor.isTerminated())
+                    throw new IllegalStateException("Failed to wait for completion");
+            }
+            catch (Exception te) {
+                throw new IllegalStateException("Failed to wait for completion", te);
+            }
+
+            return null;
+        }
+    }
+
+    /** */
+    public enum DeclaredBodyEnum {
+        ONE,
+        TWO {
+            /** {@inheritDoc} */
+            @Override public boolean isSupported() {
+                return false;
+            }
+        };
+
+        /**
+         * A bogus method.
+         */
+        public boolean isSupported() {
+            return true;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
index 22d3add4cb1e1..94806e25560e0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.internal.util.GridHandleTableSelfTest;
 import org.apache.ignite.internal.util.io.GridUnsafeDataInputOutputByteOrderSelfTest;
 import org.apache.ignite.internal.util.io.GridUnsafeDataOutputArraySizingSelfTest;
+import org.apache.ignite.marshaller.MarshallerEnumDeadlockMultiJvmTest;
 import org.apache.ignite.marshaller.jdk.GridJdkMarshallerSelfTest;
 import org.apache.ignite.marshaller.optimized.OptimizedMarshallerEnumSelfTest;
 import org.apache.ignite.marshaller.optimized.OptimizedMarshallerNodeFailoverTest;
@@ -65,6 +66,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         GridTestUtils.addTestIfNeeded(suite, DirectByteBufferStreamImplV2ByteOrderSelfTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, GridHandleTableSelfTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, OptimizedMarshallerPooledSelfTest.class, ignoredTests);
+        GridTestUtils.addTestIfNeeded(suite, MarshallerEnumDeadlockMultiJvmTest.class, ignoredTests);
 
         return suite;
     }

From 0228ca728033b1e63cba99226dd71d19f3817de0 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Thu, 31 May 2018 11:41:08 +0300
Subject: [PATCH 498/516] IGNITE-8502: Ignite client can hang during a rejoin

---
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 2651e93351fce..01970b044c367 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -1863,16 +1863,20 @@ private void tryJoin() throws InterruptedException {
 
             sockWriter.setSocket(joinRes.get1().socket(), joinRes.get2());
 
-            if (spi.joinTimeout > 0) {
-                final int joinCnt0 = joinCnt;
+            final long waitForJoinTimeout = spi.joinTimeout > 0
+                    ? spi.joinTimeout
+                    : spi.failureDetectionTimeout();
 
-                timer.schedule(new TimerTask() {
-                    @Override public void run() {
-                        if (joinCnt == joinCnt0 && joining())
-                            queue.add(JOIN_TIMEOUT);
-                    }
-                }, spi.joinTimeout);
-            }
+            final long joinTimeout0 = spi.joinTimeout;
+
+            final int joinCnt0 = joinCnt;
+
+            timer.schedule(new TimerTask() {
+                @Override public void run() {
+                    if (joinCnt == joinCnt0 && joining())
+                        queue.add(joinTimeout0 > 0 ? JOIN_TIMEOUT : SPI_RECONNECT_FAILED);
+                }
+            }, waitForJoinTimeout);
 
             sockReader.setSocket(joinRes.get1(), locNode.clientRouterNodeId());
         }

From f76a7c175bd3b182e88c2e1c3ac250cfc1544826 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Thu, 24 May 2018 17:40:01 +0300
Subject: [PATCH 499/516] IGNITE-8656 GridServiceProcessor does re-assignment
 even if no assignment is changed

---
 .../service/GridServiceProcessor.java         | 261 ++++++++++--------
 .../IgniteServiceDynamicCachesSelfTest.java   |   2 +-
 2 files changed, 149 insertions(+), 114 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 73c8ac39463b6..77205b811d1bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -611,7 +611,8 @@ else if (prj.predicate() == F.<ClusterNode>alwaysTrue())
      * @param dfltNodeFilter Default NodeFilter.
      * @return Future for deployment.
      */
-    private IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs, IgnitePredicate<ClusterNode> dfltNodeFilter) {
+    private IgniteInternalFuture<?> deployAll(Collection<ServiceConfiguration> cfgs,
+        IgnitePredicate<ClusterNode> dfltNodeFilter) {
         assert cfgs != null;
 
         if (!busyLock.enterBusy()) {
@@ -1192,158 +1193,192 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
         if (nodeFilter != null)
             ctx.resource().injectGeneric(nodeFilter);
 
-        int totalCnt = cfg.getTotalCount();
-        int maxPerNodeCnt = cfg.getMaxPerNodeCount();
-        String cacheName = cfg.getCacheName();
-        Object affKey = cfg.getAffinityKey();
+        GridServiceAssignmentsKey key = new GridServiceAssignmentsKey(cfg.getName());
+
+        GridServiceAssignments oldAssigns = (GridServiceAssignments)cache.get(key);
+
+        GridServiceAssignments assigns = new GridServiceAssignments(cfg, dep.nodeId(), topVer.topologyVersion());
+
+        Map<UUID, Integer> cnts = calculateAssignment(dep, topVer, oldAssigns, assigns.nodeFilter());
+
+        if (oldAssigns != null && oldAssigns.assigns().equals(cnts)) {
+            if (log.isInfoEnabled())
+                log.info("No changes are required for service deployment assignment [" +
+                    "svc=" + dep.configuration().getName() + ", topVer=" + topVer + ']');
+
+            return;
+        }
 
         while (true) {
-            GridServiceAssignments assigns = new GridServiceAssignments(cfg, dep.nodeId(), topVer.topologyVersion());
+            try (IgniteInternalTx tx = cache.txStartEx(PESSIMISTIC, REPEATABLE_READ)) {
+                oldAssigns = (GridServiceAssignments)cache.get(key);
 
-            Collection<ClusterNode> nodes;
+                cnts = calculateAssignment(dep, topVer, oldAssigns, assigns.nodeFilter());
 
-            // Call node filter outside of transaction.
-            if (affKey == null) {
-                nodes = ctx.discovery().nodes(topVer);
+                assigns.assigns(cnts);
 
-                if (assigns.nodeFilter() != null) {
-                    Collection<ClusterNode> nodes0 = new ArrayList<>();
+                if (log.isInfoEnabled())
+                    log.info("Calculated new assignments for service [svc=" + dep.configuration().getName() +
+                        ", assignment=" + assigns + ']');
 
-                    for (ClusterNode node : nodes) {
-                        if (assigns.nodeFilter().apply(node))
-                            nodes0.add(node);
-                    }
+                cache.put(key, assigns);
 
-                    nodes = nodes0;
-                }
-            }
-            else
-                nodes = null;
+                tx.commit();
 
-            try (IgniteInternalTx tx = cache.txStartEx(PESSIMISTIC, REPEATABLE_READ)) {
-                GridServiceAssignmentsKey key = new GridServiceAssignmentsKey(cfg.getName());
+                break;
+            }
+            catch (ClusterTopologyCheckedException e) {
+                if (log.isDebugEnabled())
+                    log.debug("Topology changed while reassigning (will retry): " + e.getMessage());
 
-                GridServiceAssignments oldAssigns = (GridServiceAssignments)cache.get(key);
+                U.sleep(10);
+            }
+        }
+    }
 
-                Map<UUID, Integer> cnts = new HashMap<>();
+    /**
+     * Calculates assignment for service.
+     * @param dep grid service deployment
+     * @param topVer topology version
+     * @param oldAssigns old service assignment
+     * @param nodeFilter node filter
+     * @return node to service count mapping
+     * @throws IgniteCheckedException If failed
+     */
+    private Map<UUID, Integer> calculateAssignment(GridServiceDeployment dep, AffinityTopologyVersion topVer,
+        GridServiceAssignments oldAssigns, IgnitePredicate<ClusterNode> nodeFilter) throws IgniteCheckedException {
+        int totalCnt = dep.configuration().getTotalCount();
+        int maxPerNodeCnt = dep.configuration().getMaxPerNodeCount();
+        String cacheName = dep.configuration().getCacheName();
+        Object affKey = dep.configuration().getAffinityKey();
 
-                if (affKey != null) {
-                    ClusterNode n = ctx.affinity().mapKeyToNode(cacheName, affKey, topVer);
+        Collection<ClusterNode> nodes;
 
-                    if (n != null) {
-                        int cnt = maxPerNodeCnt == 0 ? totalCnt == 0 ? 1 : totalCnt : maxPerNodeCnt;
+        // Call node filter outside of transaction.
+        if (affKey == null) {
+            nodes = ctx.discovery().nodes(topVer);
 
-                        cnts.put(n.id(), cnt);
+            if (nodeFilter != null) {
+                Collection<ClusterNode> nodes0 = new ArrayList<>();
 
-                        if (log.isInfoEnabled())
-                            log.info("Assigned service to primary node [svc=" + dep.configuration().getName() +
-                                ", topVer=" + topVer + ", node=" + n.id() + ']');
-                    }
+                for (ClusterNode node : nodes) {
+                    if (nodeFilter.apply(node))
+                        nodes0.add(node);
                 }
-                else {
-                    if (!nodes.isEmpty()) {
-                        int size = nodes.size();
-
-                        if (log.isInfoEnabled())
-                            log.info("Calculating assignments for service " +
-                                "[svc=" + dep.configuration().getName() +
-                                ", topVer=" + topVer +
-                                ", nodes=" + U.nodeIds(nodes) +
-                                ", oldAssignment=" + (oldAssigns == null ? "NA" : oldAssigns.assigns()) +
-                                ", totalCnt=" + totalCnt + ", maxPerNodeCnt=" + maxPerNodeCnt + ']');
-
-                        int perNodeCnt = totalCnt != 0 ? totalCnt / size : maxPerNodeCnt;
-                        int remainder = totalCnt != 0 ? totalCnt % size : 0;
-
-                        if (perNodeCnt >= maxPerNodeCnt && maxPerNodeCnt != 0) {
-                            perNodeCnt = maxPerNodeCnt;
-                            remainder = 0;
-                        }
 
-                        for (ClusterNode n : nodes)
-                            cnts.put(n.id(), perNodeCnt);
+                nodes = nodes0;
+            }
+        }
+        else
+            nodes = null;
 
-                        assert perNodeCnt >= 0;
-                        assert remainder >= 0;
 
-                        if (remainder > 0) {
-                            int cnt = perNodeCnt + 1;
+        Map<UUID, Integer> cnts = new HashMap<>();
 
-                            if (oldAssigns != null) {
-                                Collection<UUID> used = new HashSet<>();
+        if (affKey != null) {
+            ClusterNode n = ctx.affinity().mapKeyToNode(cacheName, affKey, topVer);
 
-                                // Avoid redundant moving of services.
-                                for (Map.Entry<UUID, Integer> e : oldAssigns.assigns().entrySet()) {
-                                    // Do not assign services to left nodes.
-                                    if (ctx.discovery().node(e.getKey()) == null)
-                                        continue;
+            if (n != null) {
+                int cnt = maxPerNodeCnt == 0 ? totalCnt == 0 ? 1 : totalCnt : maxPerNodeCnt;
 
-                                    // If old count and new count match, then reuse the assignment.
-                                    if (e.getValue() == cnt) {
-                                        cnts.put(e.getKey(), cnt);
+                cnts.put(n.id(), cnt);
 
-                                        used.add(e.getKey());
+                if (log.isInfoEnabled())
+                    log.info("Assigned service to primary node [svc=" + dep.configuration().getName() +
+                        ", topVer=" + topVer + ", node=" + n.id() + ']');
+            }
+        }
+        else {
+            if (!nodes.isEmpty()) {
+                int size = nodes.size();
 
-                                        if (--remainder == 0)
-                                            break;
-                                    }
-                                }
+                if (log.isInfoEnabled())
+                    log.info("Calculating assignments for service " +
+                        "[svc=" + dep.configuration().getName() +
+                        ", topVer=" + topVer +
+                        ", nodes=" + U.nodeIds(nodes) +
+                        ", oldAssignment=" + (oldAssigns == null ? "NA" : oldAssigns.assigns()) +
+                        ", totalCnt=" + totalCnt + ", maxPerNodeCnt=" + maxPerNodeCnt + ']');
+
+                int perNodeCnt = totalCnt != 0 ? totalCnt / size : maxPerNodeCnt;
+                int remainder = totalCnt != 0 ? totalCnt % size : 0;
+
+                if (perNodeCnt >= maxPerNodeCnt && maxPerNodeCnt != 0) {
+                    perNodeCnt = maxPerNodeCnt;
+                    remainder = 0;
+                }
 
-                                if (remainder > 0) {
-                                    List<Map.Entry<UUID, Integer>> entries = new ArrayList<>(cnts.entrySet());
+                for (ClusterNode n : nodes)
+                    cnts.put(n.id(), perNodeCnt);
 
-                                    // Randomize.
-                                    Collections.shuffle(entries);
+                assert perNodeCnt >= 0;
+                assert remainder >= 0;
 
-                                    for (Map.Entry<UUID, Integer> e : entries) {
-                                        // Assign only the ones that have not been reused from previous assignments.
-                                        if (!used.contains(e.getKey())) {
-                                            if (e.getValue() < maxPerNodeCnt || maxPerNodeCnt == 0) {
-                                                e.setValue(e.getValue() + 1);
+                if (remainder > 0) {
+                    int cnt = perNodeCnt + 1;
 
-                                                if (--remainder == 0)
-                                                    break;
-                                            }
-                                        }
-                                    }
-                                }
+                    if (oldAssigns != null) {
+                        Collection<UUID> used = new HashSet<>();
+
+                        // Avoid redundant moving of services.
+                        for (Map.Entry<UUID, Integer> e : oldAssigns.assigns().entrySet()) {
+                            // Do not assign services to left nodes.
+                            if (ctx.discovery().node(e.getKey()) == null)
+                                continue;
+
+                            // If old count and new count match, then reuse the assignment.
+                            if (e.getValue() == cnt) {
+                                cnts.put(e.getKey(), cnt);
+
+                                used.add(e.getKey());
+
+                                if (--remainder == 0)
+                                    break;
                             }
-                            else {
-                                List<Map.Entry<UUID, Integer>> entries = new ArrayList<>(cnts.entrySet());
+                        }
+
+                        if (remainder > 0) {
+                            List<Map.Entry<UUID, Integer>> entries = new ArrayList<>(cnts.entrySet());
 
-                                // Randomize.
-                                Collections.shuffle(entries);
+                            // Randomize.
+                            Collections.shuffle(entries);
 
-                                for (Map.Entry<UUID, Integer> e : entries) {
-                                    e.setValue(e.getValue() + 1);
+                            for (Map.Entry<UUID, Integer> e : entries) {
+                                // Assign only the ones that have not been reused from previous assignments.
+                                if (!used.contains(e.getKey())) {
+                                    if (e.getValue() < maxPerNodeCnt || maxPerNodeCnt == 0) {
+                                        e.setValue(e.getValue() + 1);
 
-                                    if (--remainder == 0)
-                                        break;
+                                        if (--remainder == 0)
+                                            break;
+                                    }
                                 }
                             }
                         }
                     }
-                }
+                    else {
+                        List<Map.Entry<UUID, Integer>> entries = new ArrayList<>(cnts.entrySet());
 
-                assigns.assigns(cnts);
+                        // Randomize.
+                        Collections.shuffle(entries);
 
-                if (log.isInfoEnabled())
-                    log.info("Calculated new assignments for service [svc=" + dep.configuration().getName() +
-                        ", assignment=" + assigns + ']');
+                        for (Map.Entry<UUID, Integer> e : entries) {
+                            e.setValue(e.getValue() + 1);
 
-                cache.put(key, assigns);
+                            if (--remainder == 0)
+                                break;
+                        }
+                    }
+                }
+            }
+        }
 
-                tx.commit();
+        HashMap<UUID, Integer> noZeroAssingment = new HashMap<>(cnts);
 
-                break;
-            }
-            catch (ClusterTopologyCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Topology changed while reassigning (will retry): " + e.getMessage());
+        for (Map.Entry<UUID, Integer> entry : cnts.entrySet())
+            if(entry.getValue() != 0) noZeroAssingment.put(entry.getKey(), entry.getValue());
 
-                U.sleep(10);
-            }
-        }
+        return noZeroAssingment;
     }
 
     /**
@@ -1519,7 +1554,7 @@ private Service copyAndInject(ServiceConfiguration cfg) throws IgniteCheckedExce
      * @param cancelCnt Number of contexts to cancel.
      */
     private void cancel(Iterable<ServiceContextImpl> ctxs, int cancelCnt) {
-        for (Iterator<ServiceContextImpl> it = ctxs.iterator(); it.hasNext();) {
+        for (Iterator<ServiceContextImpl> it = ctxs.iterator(); it.hasNext(); ) {
             ServiceContextImpl svcCtx = it.next();
 
             // Flip cancelled flag.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
index acd2b252d8f9d..a072dff5a3a1d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
@@ -156,7 +156,7 @@ public void testDeployCalledBeforeCacheStart() throws Exception {
                 @Override public boolean apply() {
                     return svcs.service(svcName) == null;
                 }
-            }, 10 * 1000);
+            }, 40 * 1000);
 
             assertTrue("Service was not undeployed", res);
         }

From 651073a6d9ad4fa4d0cd1bff2e9903192cab4981 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Thu, 17 May 2018 16:27:54 +0300
Subject: [PATCH 500/516] GG-13876 Merge service re-assignments during cluster
 startup

---
 .../apache/ignite/IgniteSystemProperties.java |  2 ++
 .../service/GridServiceProcessor.java         | 31 ++++++++++++++-----
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 866c76d358afb..d4508ec7e7e2d 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -524,6 +524,8 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_SERVICES_COMPATIBILITY_MODE = "IGNITE_SERVICES_COMPATIBILITY_MODE";
 
+    public static final String IGNITE_SERVICE_REASSIGN_DELAY = "IGNITE_SERVICE_REASSIGN_DELAY";
+
     /**
      * When set to {@code true} tree-based data structures - {@code TreeMap} and {@code TreeSet} - will not be
      * wrapped into special holders introduced to overcome serialization issue caused by missing {@code Comparable}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 77205b811d1bb..8f99043fcf4a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -111,9 +111,12 @@
 import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SERVICES_COMPATIBILITY_MODE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SERVICE_REASSIGN_DELAY;
+import static org.apache.ignite.IgniteSystemProperties.getInteger;
 import static org.apache.ignite.IgniteSystemProperties.getString;
 import static org.apache.ignite.configuration.DeploymentMode.ISOLATED;
 import static org.apache.ignite.configuration.DeploymentMode.PRIVATE;
+import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE;
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.UTILITY_CACHE_NAME;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
@@ -128,6 +131,9 @@ public class GridServiceProcessor extends GridProcessorAdapter {
     /** */
     public static final IgniteProductVersion LAZY_SERVICES_CFG_SINCE = IgniteProductVersion.fromString("1.5.22");
 
+    /** */
+    private static final int REASSIGN_DELAY = getInteger(IGNITE_SERVICE_REASSIGN_DELAY, 30_000);
+
     /** Versions that only compatible with each other, and from 1.5.33. */
     private static final Set<IgniteProductVersion> SERVICE_TOP_CALLABLE_VER1;
 
@@ -139,7 +145,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
 
     /** */
     private static final int[] EVTS = {
-        EventType.EVT_NODE_JOINED,
+        EVT_NODE_JOINED,
         EventType.EVT_NODE_LEFT,
         EventType.EVT_NODE_FAILED,
         DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT
@@ -598,12 +604,12 @@ private PreparedConfigurations prepareServiceConfigurations(Collection<ServiceCo
     public IgniteInternalFuture<?> deployAll(ClusterGroup prj, Collection<ServiceConfiguration> cfgs) {
         if (prj == null)
             // Deploy to servers by default if no projection specified.
-            return deployAll(cfgs,  ctx.cluster().get().forServers().predicate());
+            return deployAll(cfgs, ctx.cluster().get().forServers().predicate());
         else if (prj.predicate() == F.<ClusterNode>alwaysTrue())
-            return deployAll(cfgs,  null);
+            return deployAll(cfgs, null);
         else
             // Deploy to predicate nodes by default.
-            return deployAll(cfgs,  prj.predicate());
+            return deployAll(cfgs, prj.predicate());
     }
 
     /**
@@ -1936,8 +1942,9 @@ else if (msg instanceof DynamicCacheChangeBatch) {
 
                 currTopVer = topVer;
 
-                depExe.execute(new DepRunnable() {
-                    @Override public void run0() {
+                final DepRunnable depRunnable = new DepRunnable() {
+                    @Override
+                    public void run0() {
                         // In case the cache instance isn't tracked by DiscoveryManager anymore.
                         discoCache.updateAlives(ctx.discovery());
 
@@ -2031,7 +2038,17 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                             }
                         }
                     }
-                });
+                };
+
+                if (REASSIGN_DELAY <= 0 || evt.type() != EVT_NODE_JOINED)
+                    depExe.execute(depRunnable);
+                else
+                    ctx.timeout().schedule(new Runnable() {
+                        @Override
+                        public void run() {
+                            depExe.execute(depRunnable);
+                        }
+                    }, REASSIGN_DELAY, -1);
             }
             finally {
                 busyLock.leaveBusy();

From 02465d7a9bb0eae0250cf24a53eaba625b08a425 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Tue, 29 May 2018 20:06:24 +0300
Subject: [PATCH 501/516] GG-13878 CME on IgniteTxStateImpl.toString call

---
 .../processors/cache/transactions/IgniteTxStateImpl.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java
index 76751de469fa6..0f3820a0370b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java
@@ -53,7 +53,7 @@ public class IgniteTxStateImpl extends IgniteTxLocalStateAdapter {
     private GridLongList activeCacheIds = new GridLongList();
 
     /** Per-transaction read map. */
-    @GridToStringInclude
+    @GridToStringExclude
     protected Map<IgniteTxKey, IgniteTxEntry> txMap;
 
     /** Read view on transaction map. */

From caaa8df77771d0faca520c5ade9932f01a6d9e72 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Tue, 29 May 2018 20:07:29 +0300
Subject: [PATCH 502/516] IGNITE-8660 Under some circumstances server node can
 re-join back to the cluster with the same id

---
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 33 ++++++++++++++-----
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index e06aa8cc15e44..9026847aca6bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -3583,36 +3583,53 @@ else if (log.isDebugEnabled())
                     }
                 }
 
-                final IgniteNodeValidationResult err = spi.getSpiContext().validateNode(node);
+                IgniteNodeValidationResult err = spi.getSpiContext().validateNode(node);
 
-                if (err != null) {
+                if(err == null) {
+                    for (Map.Entry<Long, Collection<ClusterNode>> entry : topHist.entrySet()) {
+                        if(entry.getValue().contains(node)) {
+                            String msg1 = "Node tries to re-join cluster with the same id: [node=" + node + "]." +
+                                " Ignore join request.";
+
+                            String msg2 = "Node tries to re-join cluster with the same id: [node=" + node + "].";
+
+                            err = new IgniteNodeValidationResult(node.id(), msg1, msg2);
+
+                            break;
+                        }
+                    }
+                }
+
+                final IgniteNodeValidationResult err0 = err;
+
+                if (err0 != null) {
                     if (log.isDebugEnabled())
-                        log.debug("Node validation failed [res=" + err + ", node=" + node + ']');
+                        log.debug("Node validation failed [res=" + err0 + ", node=" + node + ']');
 
                     utilityPool.execute(
                         new Runnable() {
                             @Override public void run() {
-                                boolean ping = node.id().equals(err.nodeId()) ? pingNode(node) : pingNode(err.nodeId());
+                                boolean ping = node.id().equals(err0.nodeId()) ? pingNode(node) : pingNode(err0.nodeId());
 
                                 if (!ping) {
                                     if (log.isDebugEnabled())
                                         log.debug("Conflicting node has already left, need to wait for event. " +
                                             "Will ignore join request for now since it will be recent [req=" + msg +
-                                            ", err=" + err.message() + ']');
+                                            ", err=" + err0.message() + ']');
 
                                     // Ignore join request.
                                     return;
                                 }
 
-                                LT.warn(log, err.message());
+                                LT.warn(log, err0.message());
 
                                 // Always output in debug.
                                 if (log.isDebugEnabled())
-                                    log.debug(err.message());
+                                    log.debug(err0.message());
 
                                 try {
                                     trySendMessageDirectly(node,
-                                        new TcpDiscoveryCheckFailedMessage(locNodeId, err.sendMessage()));
+                                        new TcpDiscoveryCheckFailedMessage(locNodeId, err0.sendMessage()));
                                 }
                                 catch (IgniteSpiException e) {
                                     if (log.isDebugEnabled())

From 6becb3ae64e1ca720fd169ee6ead2255c22cc858 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Thu, 17 May 2018 19:00:24 +0300
Subject: [PATCH 503/516] IGNITE-8658 Add info message for complete partition
 exchange

---
 .../GridDhtPartitionsExchangeFuture.java         | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 70311bd2914f1..9d2db75bf9a12 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1198,8 +1198,12 @@ private void sendPartitionRequest(Set<UUID> nodeIds, GridDhtPartitionsSingleRequ
 
         if (super.onDone(res, err) && realExchange) {
             if (log.isDebugEnabled())
-                log.debug("Completed partition exchange [localNode=" + cctx.localNodeId() + ", exchange= " + this +
+                log.debug("Completed partition exchange [localNode=" + cctx.localNodeId() + ", exchange=" + this +
                     "duration=" + duration() + ", durationFromInit=" + (U.currentTimeMillis() - initTs) + ']');
+            else if(log.isInfoEnabled())
+                log.info("Completed partition exchange [localNode=" + cctx.localNodeId() + ", exchange=" + shortInfo() +
+                    ", topVer=" + topologyVersion() + "duration=" + duration() +
+                    ", durationFromInit=" + (U.currentTimeMillis() - initTs) + ']');
 
             initFut.onDone(err == null);
 
@@ -2053,6 +2057,16 @@ public void exchangeActions(ExchangeActions exchActions) {
         return exchId.hashCode();
     }
 
+    /**
+     * @return Short information string.
+     */
+    public String shortInfo() {
+        return "GridDhtPartitionsExchangeFuture [topVer=" + topologyVersion() +
+            ", evt=" + (discoveryEvent() != null ? IgniteUtils.gridEventName(discoveryEvent().type()) : -1) +
+            ", evtNode=" + (discoveryEvent() != null ? discoveryEvent().eventNode() : null) +
+            ", done=" + isDone() + ']';
+    }
+
     /**
      *
      */

From b553e5ee42e8c714c5e61e9c79e0608560d93b8a Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Sun, 20 May 2018 02:42:25 +0300
Subject: [PATCH 504/516] GG-13880 Force node kill if it blocks PME

---
 .../apache/ignite/IgniteSystemProperties.java |  4 +
 .../GridCachePartitionExchangeManager.java    | 78 ++++++++++++++++---
 .../GridDhtPartitionsExchangeFuture.java      | 38 +++++++++
 3 files changed, 111 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index d4508ec7e7e2d..1575e11a96945 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -524,8 +524,12 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_SERVICES_COMPATIBILITY_MODE = "IGNITE_SERVICES_COMPATIBILITY_MODE";
 
+    /** Sets service re-assignment delay on node join in ms */
     public static final String IGNITE_SERVICE_REASSIGN_DELAY = "IGNITE_SERVICE_REASSIGN_DELAY";
 
+    /** Enables node kill if it blocks PME(experimental feature, disabled by default) */
+    public static final String FORCE_IGNITE_STOP_ON_PME_TIMEOUT = "FORCE_IGNITE_STOP_ON_PME_TIMEOUT";
+
     /**
      * When set to {@code true} tree-based data structures - {@code TreeMap} and {@code TreeSet} - will not be
      * wrapped into special holders introduced to overcome serialization issue caused by missing {@code Comparable}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 441201dfc4849..922e40baadbc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -30,11 +30,14 @@
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -42,9 +45,12 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.affinity.AffinityFunction;
+import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
@@ -95,12 +101,15 @@
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.ignite.IgniteSystemProperties.FORCE_IGNITE_STOP_ON_PME_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PRELOAD_RESEND_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_THREAD_DUMP_ON_EXCHANGE_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.getLong;
@@ -222,7 +231,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                         affinityTopologyVersion(evt),
                         evt.type());
 
-                    exchFut = exchangeFuture(exchId, evt, cache,null, null);
+                    exchFut = exchangeFuture(exchId, evt, cache, null, null);
                 }
                 else {
                     DiscoveryCustomEvent customEvt = (DiscoveryCustomEvent)evt;
@@ -851,8 +860,8 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage(Collection<Clust
         @Nullable GridCacheVersion lastVer,
         boolean compress) {
         GridDhtPartitionsFullMessage m = new GridDhtPartitionsFullMessage(exchId,
-                lastVer,
-                exchId != null ? exchId.topologyVersion() : AffinityTopologyVersion.NONE);
+            lastVer,
+            exchId != null ? exchId.topologyVersion() : AffinityTopologyVersion.NONE);
 
         boolean useOldApi = false;
 
@@ -1006,8 +1015,7 @@ private void sendLocalPartitions(ClusterNode node, @Nullable GridDhtPartitionExc
     public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(ClusterNode targetNode,
         @Nullable GridDhtPartitionExchangeId exchangeId,
         boolean clientOnlyExchange,
-        boolean sndCounters)
-    {
+        boolean sndCounters) {
         boolean compress = canUsePartitionMapCompression(targetNode);
 
         GridDhtPartitionsSingleMessage m = new GridDhtPartitionsSingleMessage(exchangeId,
@@ -1015,7 +1023,7 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(ClusterNode
             cctx.versions().last(),
             compress);
 
-        Map<Object, T2<Integer,Map<Integer, GridDhtPartitionState>>> dupData = new HashMap<>();
+        Map<Object, T2<Integer, Map<Integer, GridDhtPartitionState>>> dupData = new HashMap<>();
 
         for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
             if (!cacheCtx.isLocal()) {
@@ -1720,6 +1728,8 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
 
                             int dumpedObjects = 0;
 
+                            int timeoutCnt = 0;
+
                             while (true) {
                                 try {
                                     exchFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);
@@ -1727,6 +1737,8 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                     break;
                                 }
                                 catch (IgniteFutureTimeoutCheckedException ignored) {
+                                    timeoutCnt++;
+
                                     U.warn(log, "Failed to wait for partition map exchange [" +
                                         "topVer=" + exchFut.topologyVersion() +
                                         ", node=" + cctx.localNodeId() + "]. " +
@@ -1745,6 +1757,30 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
 
                                         dumpedObjects++;
                                     }
+
+                                    if (timeoutCnt >= 3 && IgniteSystemProperties.getBoolean(FORCE_IGNITE_STOP_ON_PME_TIMEOUT)) {
+                                        if (exchFut.isCoordinator()) {
+                                            Set<UUID> remaining = exchFut.getRemaining();
+                                            if (remaining.isEmpty()) {
+                                                U.warn(log, "FORCE_IGNITE_STOP_ON_PME_TIMEOUT flag is enabled. Force ignite stop.");
+
+                                                Ignition.stop(cctx.kernalContext().gridName(), true);
+                                            }
+                                            else {
+                                                if (remaining.size() == 1) {
+                                                    UUID nodeToKill = remaining.iterator().next();
+
+                                                    U.warn(log, "FORCE_IGNITE_STOP_ON_PME_TIMEOUT flag is enabled. Force remote node stop: " + nodeToKill);
+
+                                                    Ignite ignite = Ignition.localIgnite();
+
+                                                    ignite.compute(ignite.cluster().forNodeId(nodeToKill)).withNoFailover().withAsync().run(
+                                                        new PoisonTask()
+                                                    );
+                                                }
+                                            }
+                                        }
+                                    }
                                 }
                                 catch (Exception e) {
                                     if (exchFut.reconnectOnError(e))
@@ -1754,7 +1790,6 @@ void addFuture(GridDhtPartitionsExchangeFuture exchFut) {
                                 }
                             }
 
-
                             if (log.isDebugEnabled())
                                 log.debug("After waiting for exchange future [exchFut=" + exchFut + ", worker=" +
                                     this + ']');
@@ -1898,7 +1933,7 @@ else if (r != null) {
                 catch (IgniteClientDisconnectedCheckedException | IgniteNeedReconnectException e) {
                     assert cctx.discovery().reconnectSupported();
 
-                    U.warn(log,"Local node failed to complete partition map exchange due to " +
+                    U.warn(log, "Local node failed to complete partition map exchange due to " +
                         "network issues, will try to reconnect to cluster", e);
 
                     cctx.discovery().reconnect();
@@ -2056,7 +2091,7 @@ private abstract class MessageHandler<M> implements IgniteBiInClosure<UUID, M> {
             if (log.isDebugEnabled())
                 log.debug("Received message from node [node=" + nodeId + ", msg=" + msg + ']');
 
-            onMessage(node , msg);
+            onMessage(node, msg);
         }
 
         /**
@@ -2101,4 +2136,29 @@ private AffinityReadyFuture(AffinityTopologyVersion topVer) {
             return S.toString(AffinityReadyFuture.class, this, super.toString());
         }
     }
+
+    /** Poison task to stop node */
+    private static class PoisonTask implements IgniteRunnable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        @IgniteInstanceResource
+        Ignite ign;
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            ExecutorService exec = Executors.newSingleThreadExecutor();
+
+            final Ignite ign0 = ign;
+
+            exec.submit(new Runnable() {
+                @Override public void run() {
+                    ign0.close();
+                }
+            });
+
+            exec.shutdown();
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 9d2db75bf9a12..463bcbc24068a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.Ignition;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.CacheEvent;
 import org.apache.ignite.events.DiscoveryEvent;
@@ -83,6 +84,7 @@
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
+import static org.apache.ignite.IgniteSystemProperties.FORCE_IGNITE_STOP_ON_PME_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_THREAD_DUMP_ON_EXCHANGE_TIMEOUT;
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
@@ -103,6 +105,10 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
     /** */
     private static final long serialVersionUID = 0L;
 
+    /** flag to force node stop if it blocks PME */
+    private static final boolean FORCE_IGNITE_STOP_OM_PME_TIMEOUT =
+        IgniteSystemProperties.getBoolean(FORCE_IGNITE_STOP_ON_PME_TIMEOUT);
+
     /** Dummy flag. */
     private final boolean dummy;
 
@@ -856,6 +862,8 @@ private void waitPartitionRelease() throws IgniteCheckedException {
 
         int dumpedObjects = 0;
 
+        int timeoutCnt = 0;
+
         while (true) {
             try {
                 partReleaseFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);
@@ -863,12 +871,20 @@ private void waitPartitionRelease() throws IgniteCheckedException {
                 break;
             }
             catch (IgniteFutureTimeoutCheckedException ignored) {
+                timeoutCnt++;
+
                 // Print pending transactions and locks that might have led to hang.
                 if (dumpedObjects < DUMP_PENDING_OBJECTS_THRESHOLD) {
                     dumpPendingObjects();
 
                     dumpedObjects++;
                 }
+
+                if (timeoutCnt >= 3 && FORCE_IGNITE_STOP_OM_PME_TIMEOUT) {
+                    U.warn(log, "FORCE_IGNITE_STOP_ON_PME_TIMEOUT flag is enabled. Force ignite stop.");
+
+                    Ignition.stop(cctx.kernalContext().gridName(), true);
+                }
             }
         }
 
@@ -879,6 +895,8 @@ private void waitPartitionRelease() throws IgniteCheckedException {
 
         dumpedObjects = 0;
 
+        timeoutCnt = 0;
+
         while (true) {
             try {
                 locksFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);
@@ -886,6 +904,8 @@ private void waitPartitionRelease() throws IgniteCheckedException {
                 break;
             }
             catch (IgniteFutureTimeoutCheckedException ignored) {
+                timeoutCnt++;
+
                 if (dumpedObjects < DUMP_PENDING_OBJECTS_THRESHOLD) {
                     U.warn(log, "Failed to wait for locks release future. Dumping pending objects that might be the " +
                         "cause [topVer=" + topologyVersion() + ", nodeId=" + cctx.localNodeId() + "]: ");
@@ -909,6 +929,12 @@ private void waitPartitionRelease() throws IgniteCheckedException {
                     if (IgniteSystemProperties.getBoolean(IGNITE_THREAD_DUMP_ON_EXCHANGE_TIMEOUT, false))
                         U.dumpThreads(log);
                 }
+
+                if (timeoutCnt >= 3 && FORCE_IGNITE_STOP_OM_PME_TIMEOUT) {
+                    U.warn(log, "FORCE_IGNITE_STOP_ON_PME_TIMEOUT flag is enabled. Force ignite stop.");
+
+                    Ignition.stop(cctx.kernalContext().gridName(), true);
+                }
             }
         }
     }
@@ -1885,6 +1911,18 @@ private void initDone() {
         initFut.onDone(true);
     }
 
+    public boolean isCoordinator() {
+        synchronized (mux) {
+            return crd != null ? crd.isLocal() : false;
+        }
+    }
+
+    public Set<UUID> getRemaining() {
+        synchronized (mux) {
+            return new HashSet<>(remaining);
+        }
+    }
+
     /**
      * Node left callback, processed from the same thread as {@link #onAffinityChangeMessage}.
      *

From b43304f743b6b3b65b14f30dc496965cad914cef Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Thu, 31 May 2018 11:01:30 +0300
Subject: [PATCH 505/516] GG-13874 Implement WA for TCP communication related
 to hanging on descriptor reservation.

(cherry picked from commit f2a6133)
---
 .../apache/ignite/IgniteSystemProperties.java |   6 +
 .../util/nio/GridNioRecoveryDescriptor.java   |  54 ++++++-
 .../internal/util/nio/GridNioSessionImpl.java |   5 +
 .../util/nio/GridSelectorNioSessionImpl.java  |  23 +++
 .../tcp/TcpCommunicationSpi.java              |  20 ++-
 ...nectionConcurrentReserveAndRemoveTest.java | 140 ++++++++++++++++++
 .../junits/GridAbstractTest.java              |  14 +-
 7 files changed, 255 insertions(+), 7 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 1575e11a96945..ad3a002dca40e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -629,6 +629,12 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_DEV_ONLY_LOGGING_DISABLED = "IGNITE_DEV_ONLY_LOGGING_DISABLED";
 
+    /**
+     * Sets timeout for TCP client recovery descriptor reservation.
+     */
+    public static final String IGNITE_NIO_RECOVERY_DESCRIPTOR_RESERVATION_TIMEOUT =
+        "IGNITE_NIO_RECOVERY_DESCRIPTOR_RESERVATION_TIMEOUT";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
index 6258c13287075..a4b75475a4344 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
@@ -22,16 +22,24 @@
 import java.util.Deque;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_NIO_RECOVERY_DESCRIPTOR_RESERVATION_TIMEOUT;
+
 /**
  * Recovery information for single node.
  */
 public class GridNioRecoveryDescriptor {
+    /** Timeout for outgoing recovery descriptor reservation. */
+    private static final long DESC_RESERVATION_TIMEOUT =
+        Math.max(1_000, IgniteSystemProperties.getLong(IGNITE_NIO_RECOVERY_DESCRIPTOR_RESERVATION_TIMEOUT, 5_000));
+
     /** Number of acknowledged messages. */
     private long acked;
 
@@ -80,6 +88,10 @@ public class GridNioRecoveryDescriptor {
     /** */
     private final boolean pairedConnections;
 
+    /** Session for the descriptor. */
+    @GridToStringExclude
+    private GridNioSession ses;
+
     /**
      * @param pairedConnections {@code True} if in/out connections pair is used for communication with node.
      * @param queueLimit Maximum size of unacknowledged messages queue.
@@ -271,8 +283,19 @@ public boolean nodeAlive(@Nullable ClusterNode node) {
      */
     public boolean reserve() throws InterruptedException {
         synchronized (this) {
-            while (!connected && reserved)
-                wait();
+            long t0 = System.nanoTime();
+
+            while (!connected && reserved) {
+                wait(DESC_RESERVATION_TIMEOUT);
+
+                if ((System.nanoTime() - t0) / 1_000_000 >= DESC_RESERVATION_TIMEOUT - 100) {
+                    // Dumping a descriptor.
+                    log.error("Failed to wait for recovery descriptor reservation " +
+                        "[desc=" + this + ", ses=" + ses + ']');
+
+                    return false;
+                }
+            }
 
             if (!connected) {
                 reserved = true;
@@ -304,6 +327,8 @@ public void onConnected() {
             assert reserved : this;
             assert !connected : this;
 
+            if (!node.isClient())
+                log.error("CONNECTED-----------" + node.isClient());
             connected = true;
 
             if (handshakeReq != null) {
@@ -354,6 +379,8 @@ public void release() {
         SessionWriteRequest[] futs = null;
 
         synchronized (this) {
+            ses = null;
+
             connected = false;
 
             if (handshakeReq != null) {
@@ -439,17 +466,36 @@ public int reserveCount() {
         }
     }
 
+    /**
+     * @return Current session.
+     */
+    public synchronized GridNioSession session() {
+        return ses;
+    }
+
+    /**
+     * @param ses Session.
+     */
+    public synchronized void session(GridNioSession ses) {
+        this.ses = ses;
+    }
+
     /**
      * @param reqs Requests to notify about error.
      */
     private void notifyOnNodeLeft(SessionWriteRequest[] reqs) {
         IOException e = new IOException("Failed to send message, node has left: " + node.id());
+        IgniteException cloErr = null;
 
         for (SessionWriteRequest req : reqs) {
             req.onError(e);
 
-            if (req.ackClosure() != null)
-                req.ackClosure().apply(new IgniteException(e));
+            if (req.ackClosure() != null) {
+                if (cloErr == null)
+                    cloErr = new IgniteException(e);
+
+                req.ackClosure().apply(cloErr);
+            }
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
index 7424531248c65..d2bab3ec89fce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
@@ -75,6 +75,9 @@ public class GridNioSessionImpl implements GridNioSession {
     /** Accepted flag. */
     private final boolean accepted;
 
+    /** For debug purposes. */
+    private volatile boolean markedForClose;
+
     /**
      * @param filterChain Chain.
      * @param locAddr Local address.
@@ -153,6 +156,8 @@ public GridNioSessionImpl(
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public GridNioFuture<Boolean> close() {
+        markedForClose = true;
+
         try {
             return filterChain.onSessionClose(this);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
index 66f9176f93745..b523e909bcc61 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
@@ -25,6 +25,7 @@
 import java.util.List;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -366,6 +367,8 @@ Collection<SessionWriteRequest> writeQueue() {
         assert recoveryDesc != null;
 
         outRecovery = recoveryDesc;
+
+        outRecovery.session(this);
     }
 
     /** {@inheritDoc} */
@@ -441,6 +444,26 @@ Object systemMessage() {
         return ret;
     }
 
+    /** {@inheritDoc} */
+    @Override public GridNioFuture<Boolean> close() {
+        GridNioFuture<Boolean> fut = super.close();
+
+        if (!fut.isDone()) {
+            fut.listen(fut0 -> {
+                try {
+                    fut0.get();
+                }
+                catch (IgniteCheckedException e) {
+                    log.error("Failed to close session [ses=" + GridSelectorNioSessionImpl.this + ']', e);
+                }
+            });
+        }
+        else if (fut.error() != null)
+            log.error("Failed to close session [ses=" + GridSelectorNioSessionImpl.this + ']', fut.error());
+
+        return fut;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridSelectorNioSessionImpl.class, this, super.toString());
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 7e8a0c4233389..fa9e76ae4bf81 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2459,8 +2459,16 @@ private void sendMessage0(ClusterNode node, Message msg, IgniteInClosure<IgniteE
                 }
                 while (retry);
             }
-            catch (IgniteCheckedException e) {
-                throw new IgniteSpiException("Failed to send message to remote node: " + node, e);
+            catch (Throwable t) {
+                log.error("Failed to send message to remote node [node=" + node + ", msg=" + msg + ']', t);
+
+                if (t instanceof Error)
+                    throw (Error)t;
+
+                if (t instanceof RuntimeException)
+                    throw (RuntimeException)t;
+
+                throw new IgniteSpiException("Failed to send message to remote node: " + node, t);
             }
             finally {
                 if (client != null && removeNodeClient(node.id(), client))
@@ -2959,6 +2967,14 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
                     if (!recoveryDesc.reserve()) {
                         U.closeQuiet(ch);
 
+                        // Ensure the session is closed.
+                        GridNioSession ses = recoveryDesc.session();
+
+                        if (ses != null) {
+                            while(ses.closeTime() == 0)
+                                ses.close();
+                        }
+
                         return null;
                     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java
new file mode 100644
index 0000000000000..ea1a1090765b7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
+import org.apache.ignite.internal.util.nio.GridTcpNioCommunicationClient;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ */
+public class IgniteConnectionConcurrentReserveAndRemoveTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
+
+        c.setClientMode(igniteInstanceName.startsWith("client"));
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(ipFinder);
+
+        c.setDiscoverySpi(disco);
+
+        TestRecordingCommunicationSpi spi = new TestRecordingCommunicationSpi();
+        spi.setIdleConnectionTimeout(Integer.MAX_VALUE);
+
+        c.setCommunicationSpi(spi);
+
+        return c;
+    }
+
+    /** */
+    private static final class TestClosure implements IgniteCallable<Integer> {
+        /** Serial version uid. */
+        private static final long serialVersionUid = 0L;
+
+        /** {@inheritDoc} */
+        @Override public Integer call() throws Exception {
+            return 1;
+        }
+    }
+
+
+    public void test() throws Exception {
+        IgniteEx svr = startGrid(0);
+
+        Ignite c1 = startGrid("client1");
+
+        assertTrue(c1.configuration().isClientMode());
+
+        Ignite c2 = startGrid("client2");
+
+        assertTrue(c2.configuration().isClientMode());
+
+        TestRecordingCommunicationSpi spi2 = (TestRecordingCommunicationSpi)c1.configuration().getCommunicationSpi();
+        TestRecordingCommunicationSpi spi3 = (TestRecordingCommunicationSpi)c2.configuration().getCommunicationSpi();
+
+        spi2.blockMessages(TcpCommunicationSpi.HandshakeMessage2.class, c1.name());
+
+        AtomicInteger cnt = new AtomicInteger();
+
+        cnt.getAndAdd(c1.compute(c1.cluster().forNodeId(c2.cluster().localNode().id())).call(new TestClosure()));
+
+        TcpCommunicationSpi spi1 = (TcpCommunicationSpi)c1.configuration().getCommunicationSpi();
+
+        ConcurrentMap<UUID, GridCommunicationClient[]> clientsMap = U.field(spi1, "clients");
+
+        GridCommunicationClient[] arr = clientsMap.get(c2.cluster().localNode().id());
+
+        GridTcpNioCommunicationClient client = null;
+
+        for (GridCommunicationClient c : arr) {
+            client = (GridTcpNioCommunicationClient)c;
+
+            if(client != null) {
+                log.error("AAAAAAA" + client.session().outRecoveryDescriptor());
+                assertTrue(client.session().outRecoveryDescriptor().reserved());
+
+                assertFalse(client.session().outRecoveryDescriptor().connected());
+            }
+        }
+
+        assertNotNull(client);
+
+        //spi1.failSend = true;
+
+        IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                doSleep(1000);
+
+                //spi1.failSend = false;
+
+                cnt.getAndAdd(c1.compute(c1.cluster().forNodeId(c2.cluster().localNode().id())).call(new TestClosure()));
+            }
+        }, 1, "hang-thread");
+
+        try {
+            cnt.getAndAdd(c1.compute(c1.cluster().forNodeId(c2.cluster().localNode().id())).call(new TestClosure()));
+
+            //fail();
+        }
+        catch (IgniteException e) {
+            // Expected.
+        }
+
+        fut.get();
+
+        assertEquals(3, cnt.get());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index f649e1ec98cbc..3ca0418a3fe45 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -1961,6 +1961,18 @@ public void awaitTopologyChange() throws IgniteInterruptedCheckedException {
         }
     }
 
+    /**
+     * @param millis Time to sleep.
+     */
+    public static void doSleep(long millis) {
+        try {
+            U.sleep(millis);
+        }
+        catch (Exception e) {
+            throw new IgniteException();
+        }
+    }
+
     /**
      *
      */
@@ -2261,7 +2273,7 @@ public abstract static class TestCacheRunnable<K, V> implements TestCacheCallabl
 
         /**
          * @param ignite Ignite.
-         * @param cache Cache.
+         * @param cache cache.
          */
         public abstract void run(Ignite ignite, IgniteCache<K, V> cache) throws Exception;
     }

From 017c2a709f3bb24b8be2d3931b716407f94aa240 Mon Sep 17 00:00:00 2001
From: Anton Kurbanov <antkr.dev@gmail.com>
Date: Fri, 1 Jun 2018 13:04:05 +0300
Subject: [PATCH 506/516] Compilation fix

---
 .../util/nio/GridSelectorNioSessionImpl.java     | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
index b523e909bcc61..aebd9cbf480ad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
@@ -27,9 +27,11 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentLinkedDeque8;
 
@@ -449,12 +451,14 @@ Object systemMessage() {
         GridNioFuture<Boolean> fut = super.close();
 
         if (!fut.isDone()) {
-            fut.listen(fut0 -> {
-                try {
-                    fut0.get();
-                }
-                catch (IgniteCheckedException e) {
-                    log.error("Failed to close session [ses=" + GridSelectorNioSessionImpl.this + ']', e);
+            fut.listen(new IgniteInClosure<IgniteInternalFuture<Boolean>>() {
+                @Override public void apply(IgniteInternalFuture<Boolean> fut0) {
+                    try {
+                        fut0.get();
+                    }
+                    catch (IgniteCheckedException e) {
+                        log.error("Failed to close session [ses=" + GridSelectorNioSessionImpl.this + ']', e);
+                    }
                 }
             });
         }

From e7073af0f3e8a1e65bdf162cbc3cf31b6ff029fd Mon Sep 17 00:00:00 2001
From: Anton Kurbanov <antkr.dev@gmail.com>
Date: Fri, 1 Jun 2018 13:54:20 +0300
Subject: [PATCH 507/516] Compilation fix tests

---
 .../IgniteConnectionConcurrentReserveAndRemoveTest.java     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java
index ea1a1090765b7..7b597cd35e875 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java
@@ -74,11 +74,11 @@ private static final class TestClosure implements IgniteCallable<Integer> {
     public void test() throws Exception {
         IgniteEx svr = startGrid(0);
 
-        Ignite c1 = startGrid("client1");
+        final Ignite c1 = startGrid("client1");
 
         assertTrue(c1.configuration().isClientMode());
 
-        Ignite c2 = startGrid("client2");
+        final Ignite c2 = startGrid("client2");
 
         assertTrue(c2.configuration().isClientMode());
 
@@ -87,7 +87,7 @@ public void test() throws Exception {
 
         spi2.blockMessages(TcpCommunicationSpi.HandshakeMessage2.class, c1.name());
 
-        AtomicInteger cnt = new AtomicInteger();
+        final AtomicInteger cnt = new AtomicInteger();
 
         cnt.getAndAdd(c1.compute(c1.cluster().forNodeId(c2.cluster().localNode().id())).call(new TestClosure()));
 

From 783fb867bcbd73eb008073393b8e24aa29a22b40 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Thu, 24 May 2018 19:11:48 +0300
Subject: [PATCH 508/516] srvc test fixed.

(cherry picked from commit 6f2fcdc)
---
 .../service/GridServiceProcessor.java         | 33 ++++++++-----------
 .../IgniteServiceDynamicCachesSelfTest.java   |  5 +--
 2 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 8f99043fcf4a9..c10f908b620a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -132,7 +132,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
     public static final IgniteProductVersion LAZY_SERVICES_CFG_SINCE = IgniteProductVersion.fromString("1.5.22");
 
     /** */
-    private static final int REASSIGN_DELAY = getInteger(IGNITE_SERVICE_REASSIGN_DELAY, 30_000);
+    private static final int REASSIGNMENT_DELAY = getInteger(IGNITE_SERVICE_REASSIGN_DELAY, 3_000);
 
     /** Versions that only compatible with each other, and from 1.5.33. */
     private static final Set<IgniteProductVersion> SERVICE_TOP_CALLABLE_VER1;
@@ -1205,21 +1205,22 @@ private void reassign(GridServiceDeployment dep, AffinityTopologyVersion topVer)
 
         GridServiceAssignments assigns = new GridServiceAssignments(cfg, dep.nodeId(), topVer.topologyVersion());
 
-        Map<UUID, Integer> cnts = calculateAssignment(dep, topVer, oldAssigns, assigns.nodeFilter());
-
-        if (oldAssigns != null && oldAssigns.assigns().equals(cnts)) {
-            if (log.isInfoEnabled())
-                log.info("No changes are required for service deployment assignment [" +
-                    "svc=" + dep.configuration().getName() + ", topVer=" + topVer + ']');
+        if (oldAssigns != null && cfg.getAffinityKey() == null) {
+            Map<UUID, Integer> cnts = calculateAssignment(dep, topVer, oldAssigns, assigns.nodeFilter());
+            if(oldAssigns.assigns().equals(cnts)) {
+                if (log.isInfoEnabled())
+                    log.info("No changes are required for service deployment assignment [" +
+                        "svc=" + dep.configuration().getName() + ", topVer=" + topVer + ']');
 
-            return;
+                return;
+            }
         }
 
         while (true) {
             try (IgniteInternalTx tx = cache.txStartEx(PESSIMISTIC, REPEATABLE_READ)) {
                 oldAssigns = (GridServiceAssignments)cache.get(key);
 
-                cnts = calculateAssignment(dep, topVer, oldAssigns, assigns.nodeFilter());
+                Map<UUID, Integer> cnts = calculateAssignment(dep, topVer, oldAssigns, assigns.nodeFilter());
 
                 assigns.assigns(cnts);
 
@@ -1379,7 +1380,7 @@ private Map<UUID, Integer> calculateAssignment(GridServiceDeployment dep, Affini
             }
         }
 
-        HashMap<UUID, Integer> noZeroAssingment = new HashMap<>(cnts);
+        HashMap<UUID, Integer> noZeroAssingment = new HashMap<>();
 
         for (Map.Entry<UUID, Integer> entry : cnts.entrySet())
             if(entry.getValue() != 0) noZeroAssingment.put(entry.getKey(), entry.getValue());
@@ -1393,14 +1394,6 @@ private Map<UUID, Integer> calculateAssignment(GridServiceDeployment dep, Affini
      * @param assigns Assignments.
      */
     private void redeploy(GridServiceAssignments assigns) {
-        if (assigns.topologyVersion() < ctx.discovery().topologyVersion()) {
-            if (log.isDebugEnabled())
-                log.debug("Skip outdated assignment [assigns=" + assigns +
-                    ", topVer=" + ctx.discovery().topologyVersion() + ']');
-
-            return;
-        }
-
         String svcName = assigns.name();
 
         Integer assignCnt = assigns.assigns().get(ctx.localNodeId());
@@ -2040,7 +2033,7 @@ public void run0() {
                     }
                 };
 
-                if (REASSIGN_DELAY <= 0 || evt.type() != EVT_NODE_JOINED)
+                if (REASSIGNMENT_DELAY <= 0 || evt.type() != EVT_NODE_JOINED)
                     depExe.execute(depRunnable);
                 else
                     ctx.timeout().schedule(new Runnable() {
@@ -2048,7 +2041,7 @@ public void run0() {
                         public void run() {
                             depExe.execute(depRunnable);
                         }
-                    }, REASSIGN_DELAY, -1);
+                    }, REASSIGNMENT_DELAY, -1);
             }
             finally {
                 busyLock.leaveBusy();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
index a072dff5a3a1d..ee8d26f18366d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceDynamicCachesSelfTest.java
@@ -142,9 +142,10 @@ public void testDeployCalledBeforeCacheStart() throws Exception {
         try {
             boolean res = GridTestUtils.waitForCondition(new PA() {
                 @Override public boolean apply() {
+                    System.out.println("CHECK");
                     return svcs.service(svcName) != null;
                 }
-            }, 10 * 1000);
+            }, 60 * 1000);
 
             assertTrue("Service was not deployed", res);
 
@@ -156,7 +157,7 @@ public void testDeployCalledBeforeCacheStart() throws Exception {
                 @Override public boolean apply() {
                     return svcs.service(svcName) == null;
                 }
-            }, 40 * 1000);
+            }, 60 * 1000);
 
             assertTrue("Service was not undeployed", res);
         }

From 5099482bedb9552cae787b33973ef697835b6b56 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Tue, 19 Jun 2018 19:47:45 +0300
Subject: [PATCH 509/516] IGNITE-8471 Dependencies upgraded

---
 modules/cassandra/store/pom.xml |  6 +++++-
 modules/flink/pom.xml           | 12 ++++++++++++
 modules/flume/pom.xml           |  6 ++++++
 modules/gce/pom.xml             |  6 ++++++
 modules/hadoop/pom.xml          | 32 ++++++++++++++++++++++++++++++++
 modules/spark-2.10/pom.xml      | 10 ++++++++++
 modules/spark/pom.xml           | 20 ++++++++++++++++++++
 modules/twitter/pom.xml         |  6 ++++++
 modules/yarn/pom.xml            | 20 ++++++++++++++++++++
 parent/pom.xml                  |  6 +++---
 10 files changed, 120 insertions(+), 4 deletions(-)

diff --git a/modules/cassandra/store/pom.xml b/modules/cassandra/store/pom.xml
index eaf012a0c8ef4..f12b88b5e3874 100644
--- a/modules/cassandra/store/pom.xml
+++ b/modules/cassandra/store/pom.xml
@@ -35,7 +35,7 @@
     <url>http://ignite.apache.org</url>
 
     <properties>
-        <commons-beanutils.version>1.8.3</commons-beanutils.version>
+        <commons-beanutils.version>1.9.2</commons-beanutils.version>
         <cassandra-driver.version>3.0.0</cassandra-driver.version>
         <cassandra-all.version>3.3</cassandra-all.version>
         <netty.version>4.0.33.Final</netty.version>
@@ -130,6 +130,10 @@
                     <artifactId>log4j-over-slf4j</artifactId>
                     <groupId>org.slf4j</groupId>
                 </exclusion>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
diff --git a/modules/flink/pom.xml b/modules/flink/pom.xml
index 47e7a08da0547..288fbf625fc80 100644
--- a/modules/flink/pom.xml
+++ b/modules/flink/pom.xml
@@ -74,6 +74,18 @@
                     <groupId>org.apache.zookeeper</groupId>
                     <artifactId>zookeeper</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils-bean-collections</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
diff --git a/modules/flume/pom.xml b/modules/flume/pom.xml
index d35c4522807ba..8741e386e56d5 100644
--- a/modules/flume/pom.xml
+++ b/modules/flume/pom.xml
@@ -45,6 +45,12 @@
             <groupId>org.apache.flume</groupId>
             <artifactId>flume-ng-core</artifactId>
             <version>${flume.ng.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
diff --git a/modules/gce/pom.xml b/modules/gce/pom.xml
index 89f9a8b39e269..b4593b44831bf 100644
--- a/modules/gce/pom.xml
+++ b/modules/gce/pom.xml
@@ -45,6 +45,12 @@
             <groupId>com.google.api-client</groupId>
             <artifactId>google-api-client</artifactId>
             <version>1.22.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index e5373abc2eed1..5d41314270341 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -63,24 +63,56 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-auth</artifactId>
             <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-hdfs</artifactId>
             <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-mapreduce-client-common</artifactId>
             <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
diff --git a/modules/spark-2.10/pom.xml b/modules/spark-2.10/pom.xml
index 58e2860fc0dbb..5ee2ad319ef8e 100644
--- a/modules/spark-2.10/pom.xml
+++ b/modules/spark-2.10/pom.xml
@@ -107,6 +107,16 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils-core</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index 140f6370854ac..5cafc83023d87 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -65,6 +65,12 @@
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-catalyst_2.11</artifactId>
             <version>${spark.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -89,6 +95,20 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
diff --git a/modules/twitter/pom.xml b/modules/twitter/pom.xml
index 126c5903df7cf..67bf11352b0af 100644
--- a/modules/twitter/pom.xml
+++ b/modules/twitter/pom.xml
@@ -66,6 +66,12 @@
             <groupId>com.twitter</groupId>
             <artifactId>hbc-twitter4j</artifactId>
             <version>${twitter.hbc.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml
index a8602cfa10da4..a627346e6b921 100644
--- a/modules/yarn/pom.xml
+++ b/modules/yarn/pom.xml
@@ -43,12 +43,32 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-yarn-client</artifactId>
             <version>${hadoop-yarn.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop-yarn.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
diff --git a/parent/pom.xml b/parent/pom.xml
index c34026efb7d79..450d9db1d1550 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -55,9 +55,9 @@
         <aws.sdk.bundle.version>1.10.12_1</aws.sdk.bundle.version>
         <aws.sdk.version>1.11.75</aws.sdk.version>
         <camel.version>2.16.0</camel.version>
-        <commons.beanutils.bundle.version>1.8.3_1</commons.beanutils.bundle.version>
-        <commons.beanutils.version>1.8.3</commons.beanutils.version>
-        <commons.codec.version>1.6</commons.codec.version>
+        <commons.beanutils.bundle.version>1.9.2_1</commons.beanutils.bundle.version>
+        <commons.beanutils.version>1.9.2</commons.beanutils.version>
+        <commons.codec.version>1.11</commons.codec.version>
         <commons.collections.version>3.2.2</commons.collections.version>
         <commons.lang.version>2.6</commons.lang.version>
         <cron4j.version>2.2.5</cron4j.version>

From 160bd92313698cfc4a77dad3a6c80155cf2d9483 Mon Sep 17 00:00:00 2001
From: dkarachentsev <dkarachentsev@gridgain.com>
Date: Fri, 22 Jun 2018 15:30:44 +0300
Subject: [PATCH 510/516] IGNITE-8813 Added ComputeJobResultPolicy to JobEvent
 - Fixes #4213.

(cherry picked from commit e5af599)
---
 .../org/apache/ignite/events/JobEvent.java    | 73 ++++++++++-----
 .../processors/task/GridTaskWorker.java       | 11 ++-
 ...ridEventStorageCheckAllEventsSelfTest.java | 90 ++++++++++++++++++-
 3 files changed, 145 insertions(+), 29 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/events/JobEvent.java b/modules/core/src/main/java/org/apache/ignite/events/JobEvent.java
index 13a53a7b94d3b..b279ddc4b540f 100644
--- a/modules/core/src/main/java/org/apache/ignite/events/JobEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/events/JobEvent.java
@@ -19,41 +19,44 @@
 
 import java.util.UUID;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Grid job event.
  * <p>
- * Grid events are used for notification about what happens within the grid. Note that by
- * design Ignite keeps all events generated on the local node locally and it provides
- * APIs for performing a distributed queries across multiple nodes:
+ * Grid events are used for notification about what happens within the grid. Note that by design Ignite keeps all events
+ * generated on the local node locally and it provides APIs for performing a distributed queries across multiple nodes:
  * <ul>
- *      <li>
- *          {@link org.apache.ignite.IgniteEvents#remoteQuery(org.apache.ignite.lang.IgnitePredicate, long, int...)} -
- *          asynchronously querying events occurred on the nodes specified, including remote nodes.
- *      </li>
- *      <li>
- *          {@link org.apache.ignite.IgniteEvents#localQuery(org.apache.ignite.lang.IgnitePredicate, int...)} -
- *          querying only local events stored on this local node.
- *      </li>
- *      <li>
- *          {@link org.apache.ignite.IgniteEvents#localListen(org.apache.ignite.lang.IgnitePredicate, int...)} -
- *          listening to local grid events (events from remote nodes not included).
- *      </li>
+ * <li>
+ * {@link org.apache.ignite.IgniteEvents#remoteQuery(org.apache.ignite.lang.IgnitePredicate, long, int...)} -
+ * asynchronously querying events occurred on the nodes specified, including remote nodes.
+ * </li>
+ * <li>
+ * {@link org.apache.ignite.IgniteEvents#localQuery(org.apache.ignite.lang.IgnitePredicate, int...)} - querying only
+ * local events stored on this local node.
+ * </li>
+ * <li>
+ * {@link org.apache.ignite.IgniteEvents#localListen(org.apache.ignite.lang.IgnitePredicate, int...)} - listening to
+ * local grid events (events from remote nodes not included).
+ * </li>
  * </ul>
- * User can also wait for events using method {@link org.apache.ignite.IgniteEvents#waitForLocal(org.apache.ignite.lang.IgnitePredicate, int...)}.
+ * User can also wait for events using method {@link org.apache.ignite.IgniteEvents#waitForLocal(org.apache.ignite.lang.IgnitePredicate,
+ * int...)}.
  * <h1 class="header">Events and Performance</h1>
- * Note that by default all events in Ignite are enabled and therefore generated and stored
- * by whatever event storage SPI is configured. Ignite can and often does generate thousands events per seconds
- * under the load and therefore it creates a significant additional load on the system. If these events are
- * not needed by the application this load is unnecessary and leads to significant performance degradation.
+ * Note that by default all events in Ignite are enabled and therefore generated and stored by whatever event storage
+ * SPI is configured. Ignite can and often does generate thousands events per seconds under the load and therefore it
+ * creates a significant additional load on the system. If these events are not needed by the application this load is
+ * unnecessary and leads to significant performance degradation.
  * <p>
- * It is <b>highly recommended</b> to enable only those events that your application logic requires
- * by using {@link org.apache.ignite.configuration.IgniteConfiguration#getIncludeEventTypes()} method in Ignite configuration. Note that certain
- * events are required for Ignite's internal operations and such events will still be generated but not stored by
- * event storage SPI if they are disabled in Ignite configuration.
+ * It is <b>highly recommended</b> to enable only those events that your application logic requires by using {@link
+ * org.apache.ignite.configuration.IgniteConfiguration#getIncludeEventTypes()} method in Ignite configuration. Note that
+ * certain events are required for Ignite's internal operations and such events will still be generated but not stored
+ * by event storage SPI if they are disabled in Ignite configuration.
+ *
  * @see EventType#EVT_JOB_CANCELLED
  * @see EventType#EVT_JOB_FAILED
  * @see EventType#EVT_JOB_FAILED_OVER
@@ -88,6 +91,9 @@ public class JobEvent extends EventAdapter {
     /** */
     private UUID taskSubjId;
 
+    /** */
+    private ComputeJobResultPolicy resPlc;
+
     /** {@inheritDoc} */
     @Override public String shortDisplay() {
         return name() + ": taskName=" + taskName;
@@ -225,6 +231,25 @@ public void taskSubjectId(UUID taskSubjId) {
         this.taskSubjId = taskSubjId;
     }
 
+    /**
+     * Gets job result policy. Not null for {@link EventType#EVT_JOB_RESULTED}
+     * and {@link EventType#EVT_JOB_FAILED_OVER} event types.
+     *
+     * @return Result policy.
+     */
+    @Nullable public ComputeJobResultPolicy resultPolicy() {
+        return resPlc;
+    }
+
+    /**
+     * Sets job result policy.
+     *
+     * @param resPlc New result policy.
+     */
+    public void resultPolicy(@Nullable ComputeJobResultPolicy resPlc) {
+        this.resPlc = resPlc;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(JobEvent.class, this,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
index 50e9ed3234f92..d2a302d3ac0e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
@@ -604,7 +604,7 @@ private void processMappedJobs(Map<? extends ComputeJob, ClusterNode> jobs) thro
             if (resCache)
                 sibs.add(sib);
 
-            recordJobEvent(EVT_JOB_MAPPED, jobId, node, "Job got mapped.");
+            recordJobEvent(EVT_JOB_MAPPED, jobId, node, null, "Job got mapped.");
         }
 
         synchronized (mux) {
@@ -1062,7 +1062,7 @@ private void sendRetryRequest(final long waitms, final GridJobResultImpl jRes, f
                     }
                     finally {
                         recordJobEvent(EVT_JOB_RESULTED, jobRes.getJobContext().getJobId(),
-                            jobRes.getNode(), "Job got resulted with: " + plc);
+                            jobRes.getNode(), plc, "Job got resulted with: " + plc);
                     }
 
                     if (log.isDebugEnabled())
@@ -1265,7 +1265,7 @@ private void sendFailoverRequest(GridJobResultImpl jobRes) {
 
         if (timeout > 0) {
             recordJobEvent(EVT_JOB_FAILED_OVER, jobRes.getJobContext().getJobId(),
-                jobRes.getNode(), "Job failed over.");
+                jobRes.getNode(), FAILOVER, "Job failed over.");
 
             // Send new reference to remote nodes for execution.
             sendRequest(jobRes);
@@ -1540,9 +1540,11 @@ private void recordTaskEvent(int evtType, String msg) {
      * @param evtType Event type.
      * @param jobId Job ID.
      * @param evtNode Event node.
+     * @param plc Job result policy.
      * @param msg Event message.
      */
-    private void recordJobEvent(int evtType, IgniteUuid jobId, ClusterNode evtNode, String msg) {
+    private void recordJobEvent(int evtType, IgniteUuid jobId, ClusterNode evtNode,
+        @Nullable ComputeJobResultPolicy plc, String msg) {
         if (!internal && ctx.event().isRecordable(evtType)) {
             JobEvent evt = new JobEvent();
 
@@ -1555,6 +1557,7 @@ private void recordJobEvent(int evtType, IgniteUuid jobId, ClusterNode evtNode,
             evt.jobId(jobId);
             evt.type(evtType);
             evt.taskSubjectId(ses.subjectId());
+            evt.resultPolicy(plc);
 
             ctx.event().record(evt);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridEventStorageCheckAllEventsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridEventStorageCheckAllEventsSelfTest.java
index cb04f55d82906..fe05800548448 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridEventStorageCheckAllEventsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridEventStorageCheckAllEventsSelfTest.java
@@ -22,6 +22,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
@@ -29,6 +30,7 @@
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobAdapter;
 import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
 import org.apache.ignite.compute.ComputeTaskFuture;
 import org.apache.ignite.compute.ComputeTaskMapAsync;
 import org.apache.ignite.compute.ComputeTaskSession;
@@ -53,11 +55,14 @@
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.compute.ComputeJobResultPolicy.FAILOVER;
+import static org.apache.ignite.compute.ComputeJobResultPolicy.WAIT;
 import static org.apache.ignite.events.EventType.EVT_CHECKPOINT_LOADED;
 import static org.apache.ignite.events.EventType.EVT_CHECKPOINT_REMOVED;
 import static org.apache.ignite.events.EventType.EVT_CHECKPOINT_SAVED;
 import static org.apache.ignite.events.EventType.EVT_JOB_CANCELLED;
 import static org.apache.ignite.events.EventType.EVT_JOB_FAILED;
+import static org.apache.ignite.events.EventType.EVT_JOB_FAILED_OVER;
 import static org.apache.ignite.events.EventType.EVT_JOB_FINISHED;
 import static org.apache.ignite.events.EventType.EVT_JOB_MAPPED;
 import static org.apache.ignite.events.EventType.EVT_JOB_QUEUED;
@@ -146,6 +151,9 @@ public void testCheckpointEvents() throws Exception {
         assertEvent(evts.get(8).type(), EVT_TASK_REDUCED, evts);
         assertEvent(evts.get(9).type(), EVT_TASK_FINISHED, evts);
         assertEvent(evts.get(10).type(), EVT_JOB_FINISHED, evts);
+
+        assertNotNull(((JobEvent)evts.get(7)).resultPolicy());
+        assertEquals(WAIT, ((JobEvent)evts.get(7)).resultPolicy());
     }
 
     /**
@@ -186,6 +194,9 @@ public void testTaskUndeployEvents() throws Exception {
         assertEvent(evts.get(9).type(), EVT_JOB_FINISHED, evts);
         assertEvent(evts.get(10).type(), EVT_TASK_UNDEPLOYED, evts);
         assertEvent(evts.get(11).type(), EVT_TASK_DEPLOYED, evts);
+
+        assertNotNull(((JobEvent)evts.get(6)).resultPolicy());
+        assertEquals(WAIT, ((JobEvent)evts.get(6)).resultPolicy());
     }
 
     /**
@@ -210,8 +221,52 @@ public void testSuccessTask() throws Exception {
         assertEvent(evts.get(7).type(), EVT_TASK_REDUCED, evts);
         assertEvent(evts.get(8).type(), EVT_TASK_FINISHED, evts);
         assertEvent(evts.get(9).type(), EVT_JOB_FINISHED, evts);
+
+        assertNotNull(((JobEvent)evts.get(6)).resultPolicy());
+        assertEquals(WAIT, ((JobEvent)evts.get(6)).resultPolicy());
     }
 
+    /**
+     * @throws Exception If test failed.
+     */
+    public void testFailoverJobTask() throws Exception {
+        startGrid(0);
+
+        try {
+            generateEvents(null, new GridAllEventsSuccessTestJob()).get();
+
+            ignite.compute().execute(GridFailoverTestTask.class.getName(), new GridAllEventsSuccessTestJob());
+
+            long tstamp = startTimestamp();
+
+            ignite.compute().execute(GridFailoverTestTask.class.getName(), new GridAllEventsSuccessTestJob());
+
+            List<Event> evts = pullEvents(tstamp, 12, GridFailoverTestTask.class.getName());
+
+            int cnt = 0;
+
+            assertEvent(evts.get(cnt++).type(), EVT_TASK_STARTED, evts);
+            assertEvent(evts.get(cnt++).type(), EVT_JOB_MAPPED, evts);
+            assertEvent(evts.get(cnt++).type(), EVT_JOB_RESULTED, evts);
+
+            assertEquals(((JobEvent)evts.get(cnt - 1)).resultPolicy(), FAILOVER);
+
+            assertEvent(evts.get(cnt++).type(), EVT_JOB_FAILED_OVER, evts);
+            assertEvent(evts.get(cnt++).type(), EVT_JOB_QUEUED, evts);
+            assertEvent(evts.get(cnt++).type(), EVT_JOB_STARTED, evts);
+            assertEvent(evts.get(cnt++).type(), EVT_CHECKPOINT_SAVED, evts);
+            assertEvent(evts.get(cnt++).type(), EVT_CHECKPOINT_REMOVED, evts);
+            assertEvent(evts.get(cnt++).type(), EVT_JOB_RESULTED, evts);
+            assertEvent(evts.get(cnt++).type(), EVT_TASK_REDUCED, evts);
+            assertEvent(evts.get(cnt++).type(), EVT_TASK_FINISHED, evts);
+            assertEvent(evts.get(cnt++).type(), EVT_JOB_FINISHED, evts);
+        }
+        finally {
+            stopGrid(0);
+        }
+    }
+
+
     /**
      * @throws Exception If test failed.
      */
@@ -238,6 +293,9 @@ public void testFailTask() throws Exception {
         assertEvent(evts.get(4).type(), EVT_JOB_RESULTED, evts);
         assertEvent(evts.get(5).type(), EVT_TASK_FAILED, evts);
         assertEvent(evts.get(6).type(), EVT_JOB_FAILED, evts);
+
+        // Exception was thrown, so policy is null.
+        assertNull(((JobEvent)evts.get(4)).resultPolicy());
     }
 
     /**
@@ -327,7 +385,20 @@ private long startTimestamp() throws Exception {
      * @throws Exception If failed.
      */
     private List<Event> pullEvents(long since, int evtCnt) throws Exception {
-        IgnitePredicate<Event> filter = new CustomEventFilter(GridAllEventsTestTask.class.getName(), since);
+        return pullEvents(since, evtCnt, GridAllEventsTestTask.class.getName());
+    }
+
+    /**
+     * Pull all test task related events since the given moment.
+     *
+     * @param since Earliest time to pulled events.
+     * @param evtCnt Expected event count.
+     * @param taskName Name of the task.
+     * @return List of events.
+     * @throws Exception If failed.
+     */
+    private List<Event> pullEvents(long since, int evtCnt, String taskName) throws Exception {
+        IgnitePredicate<Event> filter = new CustomEventFilter(taskName, since);
 
         for (int i = 0; i < 3; i++) {
             List<Event> evts = new ArrayList<>(ignite.events().localQuery((filter)));
@@ -503,4 +574,21 @@ private static class GridAllEventsTestTask extends ComputeTaskSplitAdapter<Objec
             return (Serializable)results;
         }
     }
+
+    /**
+     *
+     */
+    private static class GridFailoverTestTask extends GridAllEventsTestTask {
+        /** */
+        private final AtomicBoolean failed = new AtomicBoolean();
+
+        /** {@inheritDoc} */
+        @Override public ComputeJobResultPolicy result(ComputeJobResult res,
+            List<ComputeJobResult> rcvd) throws IgniteException {
+            if (failed.compareAndSet(false, true))
+                return FAILOVER;
+
+            return super.result(res, rcvd);
+        }
+    }
 }

From 926f07110df42beec9f8d90ed3d354bcbcfad54f Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Thu, 19 Jul 2018 16:11:38 +0300
Subject: [PATCH 511/516] IGNITE-8922 Fix for discovery message delivery
 guarantee can be violated - Fixes #4349.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
(cherry picked from commit e2cf5cf99907825e8e19cf162c24a816b8ffd2df)
---
 .../discovery/CustomMessageWrapper.java       |   4 +-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  23 +-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   7 +
 ...cpDiscoveryPendingMessageDeliveryTest.java | 281 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   3 +
 5 files changed, 312 insertions(+), 6 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
index 8f56248138fcc..9b029e5fa1864 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
@@ -23,7 +23,7 @@
 /**
  *
  */
-class CustomMessageWrapper implements DiscoverySpiCustomMessage {
+public class CustomMessageWrapper implements DiscoverySpiCustomMessage {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -33,7 +33,7 @@ class CustomMessageWrapper implements DiscoverySpiCustomMessage {
     /**
      * @param delegate Delegate.
      */
-    CustomMessageWrapper(DiscoveryCustomMessage delegate) {
+    public CustomMessageWrapper(DiscoveryCustomMessage delegate) {
         this.delegate = delegate;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 9026847aca6bb..667ad24503421 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -2337,12 +2337,22 @@ void add(TcpDiscoveryAbstractMessage msg) {
             msgs.add(new PendingMessage(msg));
 
             while (msgs.size() > MAX) {
-                PendingMessage polled = msgs.poll();
+                PendingMessage queueHead = msgs.peek();
 
-                assert polled != null;
+                assert queueHead != null;
 
-                if (polled.id.equals(discardId))
+                if (queueHead.customMsg && customDiscardId != null) {
+                    if (queueHead.id.equals(customDiscardId))
+                        customDiscardId = null;
+                }
+                else if (!queueHead.customMsg && discardId != null) {
+                    if (queueHead.id.equals(discardId))
+                        discardId = null;
+                }
+                else
                     break;
+
+                msgs.poll();
             }
         }
 
@@ -2657,6 +2667,8 @@ private void initConnectionCheckFrequency() {
          * @param msg Message to process.
          */
         @Override protected void processMessage(TcpDiscoveryAbstractMessage msg) {
+            spi.startMessageProcess(msg);
+
             sendHeartbeatMessage();
 
             DebugLogger log = messageLogger(msg);
@@ -5289,8 +5301,11 @@ private void processCustomMessage(TcpDiscoveryCustomEventMessage msg) {
 
                         if (sendMessageToRemotes(msg))
                             sendMessageAcrossRing(msg);
-                        else
+                        else {
+                            registerPendingMessage(msg);
+
                             processCustomMessage(msg);
+                        }
                     }
 
                     msg.message(null, msg.messageBytes());
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 5b94a8d403c5e..bc0afaeba920a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1392,6 +1392,13 @@ protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, long
         writeToSocket(sock, socketStream(sock), msg, timeout);
     }
 
+    /**
+     * @param msg Message.
+     */
+    protected void startMessageProcess(TcpDiscoveryAbstractMessage msg) {
+        // No-op, intended for usage in tests.
+    }
+
     /**
      * Writes message to the socket.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java
new file mode 100644
index 0000000000000..04493fc309798
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.util.Set;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.eclipse.jetty.util.ConcurrentHashSet;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class TcpDiscoveryPendingMessageDeliveryTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private volatile boolean blockMsgs;
+
+    /** */
+    private Set<TcpDiscoveryAbstractMessage> receivedEnsuredMsgs;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        blockMsgs = false;
+        receivedEnsuredMsgs = new ConcurrentHashSet<>();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi disco;
+
+        if (igniteInstanceName.startsWith("victim"))
+            disco = new DyingDiscoverySpi();
+        else if (igniteInstanceName.startsWith("listener"))
+            disco = new ListeningDiscoverySpi();
+        else
+            disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPendingMessagesOverflow() throws Exception {
+        Ignite coord = startGrid("coordinator");
+        final TcpDiscoverySpi coordDisco = (TcpDiscoverySpi)coord.configuration().getDiscoverySpi();
+
+        final Set<TcpDiscoveryAbstractMessage> sentEnsuredMsgs = new ConcurrentHashSet<>();
+        coordDisco.addSendMessageListener(new IgniteInClosure<TcpDiscoveryAbstractMessage>() {
+            @Override public void apply(TcpDiscoveryAbstractMessage msg) {
+                if (coordDisco.ensured(msg))
+                    sentEnsuredMsgs.add(msg);
+            }
+        });
+
+        // Victim doesn't send acknowledges, so we need an intermediate node to accept messages,
+        // so the coordinator could mark them as pending.
+        Ignite mediator = startGrid("mediator");
+
+        Ignite victim = startGrid("victim");
+
+        startGrid("listener");
+
+        sentEnsuredMsgs.clear();
+        receivedEnsuredMsgs.clear();
+
+        // Initial custom message will travel across the ring and will be discarded.
+        sendDummyCustomMessage(coordDisco, IgniteUuid.randomUuid());
+
+        assertTrue("Sent: " + sentEnsuredMsgs + "; received: " + receivedEnsuredMsgs,
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    log.info("Waiting for messages delivery");
+
+                    return receivedEnsuredMsgs.equals(sentEnsuredMsgs);
+                }
+            }, 10000));
+
+        blockMsgs = true;
+
+        log.info("Sending dummy custom messages");
+
+        // Non-discarded messages shouldn't be dropped from the queue.
+        int msgsNum = 2000;
+
+        for (int i = 0; i < msgsNum; i++)
+            sendDummyCustomMessage(coordDisco, IgniteUuid.randomUuid());
+
+        mediator.close();
+        victim.close();
+
+        assertTrue("Sent: " + sentEnsuredMsgs + "; received: " + receivedEnsuredMsgs,
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    log.info("Waiting for messages delivery");
+
+                    return receivedEnsuredMsgs.equals(sentEnsuredMsgs);
+                }
+            }, 10000));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCustomMessageInSingletonCluster() throws Exception {
+        Ignite coord = startGrid("coordinator");
+        final TcpDiscoverySpi coordDisco = (TcpDiscoverySpi)coord.configuration().getDiscoverySpi();
+
+        final Set<TcpDiscoveryAbstractMessage> sentEnsuredMsgs = new ConcurrentHashSet<>();
+        coordDisco.addSendMessageListener(new IgniteInClosure<TcpDiscoveryAbstractMessage>() {
+            @Override public void apply(TcpDiscoveryAbstractMessage msg) {
+                if (coordDisco.ensured(msg))
+                    sentEnsuredMsgs.add(msg);
+            }
+        });
+
+        // Custom message on a singleton cluster shouldn't break consistency of PendingMessages.
+        sendDummyCustomMessage(coordDisco, IgniteUuid.randomUuid());
+
+        // Victim doesn't send acknowledges, so we need an intermediate node to accept messages,
+        // so the coordinator could mark them as pending.
+        Ignite mediator = startGrid("mediator");
+
+        Ignite victim = startGrid("victim");
+
+        startGrid("listener");
+
+        sentEnsuredMsgs.clear();
+        receivedEnsuredMsgs.clear();
+
+        blockMsgs = true;
+
+        log.info("Sending dummy custom messages");
+
+        // Non-discarded messages shouldn't be dropped from the queue.
+        int msgsNum = 100;
+
+        for (int i = 0; i < msgsNum; i++)
+            sendDummyCustomMessage(coordDisco, IgniteUuid.randomUuid());
+
+        mediator.close();
+        victim.close();
+
+        assertTrue("Sent: " + sentEnsuredMsgs + "; received: " + receivedEnsuredMsgs,
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    log.info("Waiting for messages delivery");
+
+                    return receivedEnsuredMsgs.equals(sentEnsuredMsgs);
+                }
+            }, 10000));
+    }
+
+    /**
+     * @param disco Discovery SPI.
+     * @param id Message id.
+     */
+    private void sendDummyCustomMessage(TcpDiscoverySpi disco, IgniteUuid id) {
+        disco.sendCustomEvent(new CustomMessageWrapper(new DummyCustomDiscoveryMessage(id)));
+    }
+
+    /**
+     * Discovery SPI, that makes a node stop sending messages when {@code blockMsgs} is set to {@code true}.
+     */
+    private class DyingDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, byte[] data,
+            long timeout) throws IOException {
+            if (!blockMsgs)
+                super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (!blockMsgs)
+                super.writeToSocket(sock, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (!blockMsgs)
+                super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(TcpDiscoveryAbstractMessage msg, Socket sock, int res,
+            long timeout) throws IOException {
+            if (!blockMsgs)
+                super.writeToSocket(msg, sock, res, timeout);
+        }
+    }
+
+    /**
+     *
+     */
+    private class ListeningDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void startMessageProcess(TcpDiscoveryAbstractMessage msg) {
+            if (ensured(msg))
+                receivedEnsuredMsgs.add(msg);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class DummyCustomDiscoveryMessage implements DiscoveryCustomMessage {
+        /** */
+        private final IgniteUuid id;
+
+        /**
+         * @param id Message id.
+         */
+        DummyCustomDiscoveryMessage(IgniteUuid id) {
+            this.id = id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteUuid id() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public DiscoveryCustomMessage ackMessage() {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isMutable() {
+            return false;
+        }
+    }
+}
+
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 43e9153cf3084..8650914ff6e21 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeAttributesUpdateOnReconnectTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeConfigConsistentIdSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeConsistentIdSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryPendingMessageDeliveryTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryRestartTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySnapshotHistoryTest;
@@ -106,6 +107,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));
 
+        suite.addTest(new TestSuite(TcpDiscoveryPendingMessageDeliveryTest.class));
+
         return suite;
     }
 }

From 1d993a61c5b9111aa32bc776776e0c804fccc270 Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <stanlukyanov@gmail.com>
Date: Mon, 20 Aug 2018 19:25:38 +0300
Subject: [PATCH 512/516] IGNITE-9315 Fixed invalid meta for eviction policy
 when near cache context is used instead of DHT.

---
 .../processors/cache/GridCacheUtils.java      |   4 +-
 .../preloader/GridDhtPartitionDemander.java   |   4 +-
 .../eviction/DhtAndNearEvictionTest.java      | 192 ++++++++++++++++++
 .../testframework/GridStringLogger.java       |  35 +++-
 4 files changed, 223 insertions(+), 12 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/DhtAndNearEvictionTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index f68a1354c3613..2582aadaac08b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1732,7 +1732,7 @@ public static TransactionConfiguration transactionConfiguration(final @Nullable
      * <p>
      * Useful only when store with readThrough is used. In situation when
      * get() on backup node returns successful result, it's expected that
-     * localPeek() will be successful as well. But it doesn't true when
+     * localPeek() will be successful as well. But it isn't true when
      * primary node loaded value from local store, in this case backups
      * will remain non-initialized.
      * <br>
@@ -1809,7 +1809,7 @@ else if (ttl > 0) {
                     }
                     finally {
                         if (entry != null)
-                            cctx.evicts().touch(entry, topVer);
+                            entry.context().evicts().touch(entry, topVer);
                     }
                 }
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index ab2dae917d1f7..380a86f73b68d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -784,7 +784,7 @@ private boolean preloadEntry(
                         cctx.isDrEnabled() ? DR_PRELOAD : DR_NONE,
                         false
                     )) {
-                        cctx.evicts().touch(cached, topVer); // Start tracking.
+                        cached.context().evicts().touch(cached, topVer); // Start tracking.
 
                         if (cctx.events().isRecordable(EVT_CACHE_REBALANCE_OBJECT_LOADED) && !cached.isInternal())
                             cctx.events().addEvent(cached.partition(), cached.key(), cctx.localNodeId(),
@@ -793,7 +793,7 @@ private boolean preloadEntry(
                     }
                     else {
                         if (cctx.isSwapOrOffheapEnabled())
-                            cctx.evicts().touch(cached, topVer); // Start tracking.
+                            cached.context().evicts().touch(cached, topVer); // Start tracking.
 
                         if (log.isDebugEnabled())
                             log.debug("Rebalancing entry is already in cache (will ignore) [key=" + cached.key() +
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/DhtAndNearEvictionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/DhtAndNearEvictionTest.java
new file mode 100644
index 0000000000000..97d1d7833d88e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/DhtAndNearEvictionTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.eviction;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.concurrent.Callable;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.eviction.lru.LruEvictionPolicyFactory;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridStringLogger;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Checking that DHT and near cache evictions work correctly when both are set.
+ *
+ * This is a regression test for IGNITE-9315.
+ */
+public class DhtAndNearEvictionTest extends GridCommonAbstractTest {
+    /** */
+    public GridStringLogger strLog;
+
+    /** */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setGridLogger(strLog);
+
+        TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
+        ipFinder.setAddresses(Collections.singleton("127.0.0.1:47500..47501"));
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(ipFinder));
+
+        return cfg;
+    }
+
+    /** */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        strLog = new GridStringLogger(false, log);
+        strLog.logLength(1024 * 1024);
+    }
+
+    /** */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     * Checking the case that provokes usage of
+     * GridCacheUtils.createBackupPostProcessingClosure.BackupPostProcessingClosure
+     * which used to be affected by IGNITE-9315:
+     * <ul>
+     *     <li>2 nodes, one writing, one reading</li>
+     *     <li>cache store with read-through</li>
+     *     <li>backups=1</li>
+     * </ul>
+     */
+    public void testConcurrentWritesAndReadsWithReadThrough() throws Exception {
+        startGrid(0);
+        startGrid(1);
+
+        CacheConfiguration<?, ?> ccfg = new CacheConfiguration<Integer, Integer>("mycache")
+            .setEvictionPolicyFactory(new LruEvictionPolicyFactory<>(500))
+            .setNearConfiguration(
+                new NearCacheConfiguration<Integer, Integer>()
+                    .setNearEvictionPolicyFactory(new LruEvictionPolicyFactory<>(100))
+            )
+            .setCacheStoreFactory(DummyCacheStore.factoryOf())
+            .setBackups(1);
+        ccfg.setReadThrough(true);
+
+        grid(0).createCache(ccfg);
+
+        IgniteInternalFuture<?> fut1 = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                IgniteCache<Integer, Integer> cache = DhtAndNearEvictionTest.this.grid(0).cache("mycache");
+
+                for (int i = 0; i < 1000; i++)
+                    cache.put(i, i);
+
+                return null;
+            }
+        });
+
+        IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                IgniteCache<Integer, Integer> cache = DhtAndNearEvictionTest.this.grid(1).cache("mycache");
+
+                for (int i = 0; i < 1000; i++)
+                    cache.get(i);
+
+                return null;
+            }
+        });
+
+        // AssertionError may leave the node hanging.
+        // Because of that, wait until either the futures are done or the log contains an error.
+        while (!fut1.isDone() || !fut2.isDone()) {
+            assertFalse(strLog.toString().contains("AssertionError"));
+
+            Thread.sleep(1000);
+        }
+
+        fut1.get();
+        fut2.get();
+
+        assertFalse(strLog.toString().contains("AssertionError"));
+    }
+
+    /**
+     * Checking rebalancing which used to be affected by IGNITE-9315.
+     */
+    public void testRebalancing() throws Exception {
+        Ignite grid0 = startGrid(0);
+
+        CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<Integer, Integer>("mycache")
+            .setEvictionPolicyFactory(new LruEvictionPolicyFactory<>(500))
+            .setNearConfiguration(
+                new NearCacheConfiguration<Integer, Integer>()
+                    .setNearEvictionPolicyFactory(new LruEvictionPolicyFactory<>(100))
+            );
+
+        IgniteCache<Integer, Integer> cache = grid0.createCache(ccfg);
+
+        for (int i = 0; i < 1000; i++)
+            cache.put(i, i);
+
+        startGrid(1);
+
+        awaitPartitionMapExchange(true, true, null);
+
+        assertFalse(strLog.toString().contains("AssertionError"));
+    }
+
+    /** */
+    private static class DummyCacheStore extends CacheStoreAdapter<Integer, Integer> implements Serializable {
+        /** {@inheritDoc} */
+        @Override public Integer load(Integer key) throws CacheLoaderException {
+            return key;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(
+            Cache.Entry<? extends Integer, ? extends Integer> entry) throws CacheWriterException {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            // No-op.
+        }
+
+        /** */
+        public static Factory<DummyCacheStore> factoryOf() {
+            return new Factory<DummyCacheStore>() {
+                @Override public DummyCacheStore create() {
+                    return new DummyCacheStore();
+                }
+            };
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java
index da1398e875e77..1be29f032e055 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java
@@ -37,6 +37,9 @@ public class GridStringLogger implements IgniteLogger {
     /** */
     private final boolean dbg;
 
+    /** */
+    private volatile int chars = CHAR_CNT;
+
     /** */
     private final IgniteLogger echo;
 
@@ -63,23 +66,39 @@ public GridStringLogger(boolean dbg, @Nullable IgniteLogger echo) {
         this.echo = echo;
     }
 
+    /**
+     * @param chars History buffer length.
+     */
+    public void logLength(int chars) {
+        this.chars = chars;
+    }
+
+    /**
+     * @return History buffer length.
+     */
+    private int logLength() {
+        return chars;
+    }
+
     /**
      * @param msg Message to log.
      */
-    private void log(String msg) {
+    private synchronized void log(String msg) {
         buf.append(msg).append(U.nl());
 
         if (echo != null)
             echo.info("[GridStringLogger echo] " + msg);
 
-        if (buf.length() > CHAR_CNT) {
+        int logLength = logLength();
+
+        if (buf.length() > logLength) {
             if (echo != null)
                 echo.warning("Cleaning GridStringLogger history.");
 
-            buf.delete(0, buf.length() - CHAR_CNT);
+            buf.delete(0, buf.length() - logLength);
         }
 
-        assert buf.length() <= CHAR_CNT;
+        assert buf.length() <= logLength;
     }
 
     /** {@inheritDoc} */
@@ -123,7 +142,7 @@ private void log(String msg) {
     }
 
     /** {@inheritDoc} */
-    @Override public void warning(String msg, @Nullable Throwable e) {
+    @Override public synchronized void warning(String msg, @Nullable Throwable e) {
         log(msg);
 
         if (e != null)
@@ -141,7 +160,7 @@ private void log(String msg) {
     }
 
     /** {@inheritDoc} */
-    @Override public void error(String msg, @Nullable Throwable e) {
+    @Override public synchronized void error(String msg, @Nullable Throwable e) {
         log(msg);
 
         if (e != null)
@@ -181,12 +200,12 @@ private void log(String msg) {
     /**
      * Resets logger.
      */
-    public void reset() {
+    public synchronized void reset() {
         buf.setLength(0);
     }
 
     /** {@inheritDoc} */
-    @Override public String toString() {
+    @Override public synchronized String toString() {
         return buf.toString();
     }
 }
\ No newline at end of file

From 187beb9d17a51f412e02c346e288e02af52418e1 Mon Sep 17 00:00:00 2001
From: Denis Loginov <da.loginov@jet.msk.su>
Date: Mon, 11 Dec 2017 16:16:07 +0300
Subject: [PATCH 513/516] IGNITE-5731: curWaitingJobs and avgJobWaitTime
 calculation fixed

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit 321ba60)
---
 .../org/apache/ignite/internal/ClusterMetricsSnapshot.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/ClusterMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/ClusterMetricsSnapshot.java
index 8a0bc5ef63bb2..ad1539f7af707 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/ClusterMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/ClusterMetricsSnapshot.java
@@ -347,7 +347,7 @@ public ClusterMetricsSnapshot(ClusterGroup p) {
             maxRejectedJobs = max(maxRejectedJobs, m.getCurrentRejectedJobs());
             avgRejectedJobs += m.getCurrentRejectedJobs();
 
-            curWaitingJobs += m.getCurrentJobWaitTime();
+            curWaitingJobs += m.getCurrentWaitingJobs();
             maxWaitingJobs = max(maxWaitingJobs, m.getCurrentWaitingJobs());
             avgWaitingJobs += m.getCurrentWaitingJobs();
 
@@ -357,7 +357,7 @@ public ClusterMetricsSnapshot(ClusterGroup p) {
 
             curJobWaitTime = min(curJobWaitTime, m.getCurrentJobWaitTime());
             maxJobWaitTime = max(maxJobWaitTime, m.getCurrentJobWaitTime());
-            avgJobWaitTime += m.getCurrentJobWaitTime();
+            avgJobWaitTime += m.getAverageJobWaitTime();
 
             daemonThreadCnt += m.getCurrentDaemonThreadCount();
 

From e93fab4a84e429f728a5b7f4ba4180af2f630317 Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Wed, 26 Sep 2018 20:11:36 +0300
Subject: [PATCH 514/516] GG-14240 Add system flag to disable IGNITE-4828

(cherry picked from commit bba4828)
---
 .../java/org/apache/ignite/IgniteSystemProperties.java     | 7 +++++++
 .../affinity/rendezvous/RendezvousAffinityFunction.java    | 6 +++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index ad3a002dca40e..9fd9d84fa81b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -617,6 +617,13 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION = "IGNITE_SUPPORT_SINGLETON_COLLECTION_SERIALIZATION";
 
+    /**
+     * Property disable changes introduced by IGNITE-4828. The change changes the way RendezvousAffinityFunction calculates
+     * partitions for key, as result, all versions with IGNITE-4828 incompatible with all older versions without IGNITE-4828.
+     * This property disable IGNITE-4828 and allow to use new and old nodes simultaneously.
+     */
+    public static final String IGNITE_OLD_PARTITION_RESOLVER = "IGNITE_OLD_PARTITION_RESOLVER";
+
     /**
      * Class name of the closure {@link IgniteClosure}<String, Throwable> that
      * will be invoked once per topology and validates cache. If it returns or throws exception,
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
index 54df331948165..1affd0ba7d069 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cache.affinity.AffinityFunctionContext;
 import org.apache.ignite.cache.affinity.AffinityNodeHashResolver;
@@ -52,6 +53,8 @@
 import org.apache.ignite.resources.LoggerResource;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_OLD_PARTITION_RESOLVER;
+
 /**
  * Affinity function for partitioned cache based on Highest Random Weight algorithm.
  * This function supports the following configuration:
@@ -234,7 +237,8 @@ public void setPartitions(int parts) {
 
         this.parts = parts;
 
-        mask = (parts & (parts - 1)) == 0 ? parts - 1 : -1;
+        mask = (parts & (parts - 1)) == 0 && !IgniteSystemProperties.getBoolean(IGNITE_OLD_PARTITION_RESOLVER)
+            ? parts - 1 : -1;
     }
 
     /**

From 46c106ccc842325cd6655c9f8e7ef6dfc61f55f5 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Wed, 24 Oct 2018 12:24:46 +0300
Subject: [PATCH 515/516] IGNITE-9899 Notify CQ filter on backups - Fixes
 #5011.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 0d7535340b61df0c327fd6afe9d3519a248ee350)
---
 .../processors/cache/GridCacheMapEntry.java   | 13 +++----
 ...ontinuousQueryAsyncFailoverTxSelfTest.java |  5 ---
 ...acheContinuousQueryFailoverTxSelfTest.java |  5 ---
 ...tinuousQueryOperationFromCallbackTest.java | 36 +++++++++++++++++++
 4 files changed, 41 insertions(+), 18 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index ba7b3680d02dd..94dae682b260a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -1261,7 +1261,7 @@ protected void recordNodeId(UUID nodeId, AffinityTopologyVersion topVer) {
             boolean internal = isInternal() || !context().userCache();
 
             Map<UUID, CacheContinuousQueryListener> lsnrCol =
-                notifyContinuousQueries(tx) ? cctx.continuousQueries().updateListeners(internal, false) : null;
+                notifyContinuousQueries() ? cctx.continuousQueries().updateListeners(internal, false) : null;
 
             old = oldValPresent ? oldVal :
                 (retval || intercept || lsnrCol != null) ?
@@ -1469,7 +1469,7 @@ protected Object keyValue(boolean cpy) {
             boolean internal = isInternal() || !context().userCache();
 
             Map<UUID, CacheContinuousQueryListener> lsnrCol =
-                notifyContinuousQueries(tx) ? cctx.continuousQueries().updateListeners(internal, false) : null;
+                notifyContinuousQueries() ? cctx.continuousQueries().updateListeners(internal, false) : null;
 
             old = oldValPresent ? oldVal : (retval || intercept || lsnrCol != null) ?
                 rawGetOrUnmarshalUnlocked(!retval && !isOffHeapValuesOnly()) : val;
@@ -1630,13 +1630,10 @@ else if (log.isDebugEnabled())
     }
 
     /**
-     * @param tx Transaction.
-     * @return {@code True} if should notify continuous query manager.
+     * @return {@code True} if should notify continuous query manager on updates of this entry.
      */
-    private boolean notifyContinuousQueries(@Nullable IgniteInternalTx tx) {
-        return cctx.isLocal() ||
-            cctx.isReplicated() ||
-            (!isNear() && !(tx != null && tx.onePhaseCommit() && !tx.local()));
+    private boolean notifyContinuousQueries() {
+        return !isNear();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverTxSelfTest.java
index 900abc8aa9470..8f0bd0e0d4633 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverTxSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverTxSelfTest.java
@@ -41,9 +41,4 @@ public class CacheContinuousQueryAsyncFailoverTxSelfTest extends CacheContinuous
     @Override protected boolean asyncCallback() {
         return true;
     }
-
-    /** {@inheritDoc} */
-    public void testNoEventLossOnTopologyChange() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-4015");
-    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxSelfTest.java
index c5240da3f524a..789a105e02b05 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxSelfTest.java
@@ -36,9 +36,4 @@ public class CacheContinuousQueryFailoverTxSelfTest extends CacheContinuousQuery
     @Override protected CacheAtomicityMode atomicityMode() {
         return TRANSACTIONAL;
     }
-
-    /** {@inheritDoc} */
-    public void testNoEventLossOnTopologyChange() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-4015");
-    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOperationFromCallbackTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOperationFromCallbackTest.java
index 0d027a96f9c6d..aeb0a6cb6b919 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOperationFromCallbackTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOperationFromCallbackTest.java
@@ -141,6 +141,42 @@ public class CacheContinuousQueryOperationFromCallbackTest extends GridCommonAbs
         filterCbCntr.set(0);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAtomicOneBackup() throws Exception {
+        CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 1, ATOMIC, FULL_SYNC);
+
+        doTest(ccfg, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxOneBackupFilter() throws Exception {
+        CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 1, TRANSACTIONAL, FULL_SYNC);
+
+        doTest(ccfg, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxOneBackupFilterPrimary() throws Exception {
+        CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 1, TRANSACTIONAL, PRIMARY_SYNC);
+
+        doTest(ccfg, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTxOneBackup() throws Exception {
+        CacheConfiguration<Object, Object> ccfg = cacheConfiguration(PARTITIONED, 1, TRANSACTIONAL, FULL_SYNC);
+
+        doTest(ccfg, true);
+    }
+
     /**
      * @throws Exception If failed.
      */

From 3acef09dc90df99247d2eda58ff2db6809dbe891 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Thu, 8 Nov 2018 11:57:25 +0300
Subject: [PATCH 516/516] Revert "IGNITE-8502: Ignite client can hang during a
 rejoin"

This reverts commit 0228ca728033b1e63cba99226dd71d19f3817de0.
---
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 22 ++++++++-----------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 01970b044c367..2651e93351fce 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -1863,20 +1863,16 @@ private void tryJoin() throws InterruptedException {
 
             sockWriter.setSocket(joinRes.get1().socket(), joinRes.get2());
 
-            final long waitForJoinTimeout = spi.joinTimeout > 0
-                    ? spi.joinTimeout
-                    : spi.failureDetectionTimeout();
+            if (spi.joinTimeout > 0) {
+                final int joinCnt0 = joinCnt;
 
-            final long joinTimeout0 = spi.joinTimeout;
-
-            final int joinCnt0 = joinCnt;
-
-            timer.schedule(new TimerTask() {
-                @Override public void run() {
-                    if (joinCnt == joinCnt0 && joining())
-                        queue.add(joinTimeout0 > 0 ? JOIN_TIMEOUT : SPI_RECONNECT_FAILED);
-                }
-            }, waitForJoinTimeout);
+                timer.schedule(new TimerTask() {
+                    @Override public void run() {
+                        if (joinCnt == joinCnt0 && joining())
+                            queue.add(JOIN_TIMEOUT);
+                    }
+                }, spi.joinTimeout);
+            }
 
             sockReader.setSocket(joinRes.get1(), locNode.clientRouterNodeId());
         }